2024-12-05 00:41:01,549 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 00:41:01,563 main DEBUG Took 0.011410 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 00:41:01,563 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 00:41:01,564 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 00:41:01,565 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 00:41:01,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,575 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 00:41:01,590 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,593 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,595 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,595 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,596 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,597 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,598 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,598 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,599 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,600 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,601 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,602 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 00:41:01,602 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,603 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 00:41:01,604 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 00:41:01,606 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 00:41:01,608 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 00:41:01,609 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 00:41:01,610 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 00:41:01,610 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 00:41:01,619 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 00:41:01,622 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 00:41:01,624 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 00:41:01,624 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 00:41:01,625 main DEBUG createAppenders(={Console}) 2024-12-05 00:41:01,626 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-05 00:41:01,626 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 00:41:01,626 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-05 00:41:01,627 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 00:41:01,627 main DEBUG OutputStream closed 2024-12-05 00:41:01,627 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 00:41:01,628 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 00:41:01,628 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-05 00:41:01,696 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 00:41:01,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 00:41:01,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 00:41:01,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 00:41:01,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 00:41:01,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 00:41:01,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 00:41:01,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 00:41:01,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 00:41:01,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 00:41:01,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 00:41:01,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 00:41:01,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 00:41:01,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 00:41:01,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 00:41:01,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 00:41:01,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 00:41:01,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 00:41:01,706 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 00:41:01,706 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-05 00:41:01,707 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 00:41:01,707 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-05T00:41:01,920 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f 2024-12-05 00:41:01,922 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 00:41:01,923 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T00:41:01,930 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-05T00:41:01,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T00:41:01,969 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c, deleteOnExit=true 2024-12-05T00:41:01,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T00:41:01,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/test.cache.data in system properties and HBase conf 2024-12-05T00:41:01,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T00:41:01,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/hadoop.log.dir in system properties and HBase conf 2024-12-05T00:41:01,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T00:41:01,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T00:41:01,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T00:41:02,059 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T00:41:02,143 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T00:41:02,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T00:41:02,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T00:41:02,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T00:41:02,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T00:41:02,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T00:41:02,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T00:41:02,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T00:41:02,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T00:41:02,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T00:41:02,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/nfs.dump.dir in system properties and HBase conf 2024-12-05T00:41:02,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/java.io.tmpdir in system properties and HBase conf 2024-12-05T00:41:02,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T00:41:02,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T00:41:02,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T00:41:03,228 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T00:41:03,294 INFO [Time-limited test {}] log.Log(170): Logging initialized @2361ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T00:41:03,367 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T00:41:03,424 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T00:41:03,443 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T00:41:03,444 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T00:41:03,445 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T00:41:03,459 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T00:41:03,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/hadoop.log.dir/,AVAILABLE} 2024-12-05T00:41:03,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T00:41:03,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/java.io.tmpdir/jetty-localhost-38621-hadoop-hdfs-3_4_1-tests_jar-_-any-4366563102412883826/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T00:41:03,627 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:38621} 2024-12-05T00:41:03,627 INFO [Time-limited test {}] server.Server(415): Started @2695ms 2024-12-05T00:41:04,167 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T00:41:04,174 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T00:41:04,175 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T00:41:04,175 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T00:41:04,176 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T00:41:04,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/hadoop.log.dir/,AVAILABLE} 2024-12-05T00:41:04,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T00:41:04,270 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/java.io.tmpdir/jetty-localhost-44283-hadoop-hdfs-3_4_1-tests_jar-_-any-12779381341492910958/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T00:41:04,270 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:44283} 2024-12-05T00:41:04,271 INFO [Time-limited test {}] server.Server(415): Started @3338ms 2024-12-05T00:41:04,317 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T00:41:05,499 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/data/data1/current/BP-1145415758-172.17.0.2-1733359262674/current, will proceed with Du for space computation calculation, 2024-12-05T00:41:05,499 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/data/data2/current/BP-1145415758-172.17.0.2-1733359262674/current, will proceed with Du for space computation calculation, 2024-12-05T00:41:05,533 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T00:41:05,594 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f 2024-12-05T00:41:05,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x408df8d7e6cf1f98 with lease ID 0x1f7d543c445f8420: Processing first storage report for DS-a5176426-ce12-425d-8b24-dcfe4287886a from datanode DatanodeRegistration(127.0.0.1:41735, datanodeUuid=e92b3860-bd6b-47f0-9a8c-8a2b489e331f, infoPort=42163, infoSecurePort=0, ipcPort=33837, storageInfo=lv=-57;cid=testClusterID;nsid=1569804562;c=1733359262674) 2024-12-05T00:41:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x408df8d7e6cf1f98 with lease ID 0x1f7d543c445f8420: from storage DS-a5176426-ce12-425d-8b24-dcfe4287886a node DatanodeRegistration(127.0.0.1:41735, datanodeUuid=e92b3860-bd6b-47f0-9a8c-8a2b489e331f, infoPort=42163, infoSecurePort=0, ipcPort=33837, storageInfo=lv=-57;cid=testClusterID;nsid=1569804562;c=1733359262674), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-05T00:41:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x408df8d7e6cf1f98 with lease ID 0x1f7d543c445f8420: Processing first storage report for DS-520f8570-a9b9-4c8a-9ffe-fc374d5d7921 from datanode DatanodeRegistration(127.0.0.1:41735, datanodeUuid=e92b3860-bd6b-47f0-9a8c-8a2b489e331f, infoPort=42163, infoSecurePort=0, ipcPort=33837, storageInfo=lv=-57;cid=testClusterID;nsid=1569804562;c=1733359262674) 2024-12-05T00:41:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x408df8d7e6cf1f98 with lease ID 0x1f7d543c445f8420: from storage DS-520f8570-a9b9-4c8a-9ffe-fc374d5d7921 node DatanodeRegistration(127.0.0.1:41735, datanodeUuid=e92b3860-bd6b-47f0-9a8c-8a2b489e331f, infoPort=42163, infoSecurePort=0, ipcPort=33837, storageInfo=lv=-57;cid=testClusterID;nsid=1569804562;c=1733359262674), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T00:41:05,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/zookeeper_0, clientPort=50354, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T00:41:05,696 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50354 2024-12-05T00:41:05,711 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:05,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:05,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741825_1001 (size=7) 2024-12-05T00:41:06,350 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 with version=8 2024-12-05T00:41:06,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/hbase-staging 2024-12-05T00:41:06,437 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T00:41:06,670 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:41:06,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:41:06,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:41:06,687 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:41:06,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:41:06,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:41:06,801 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T00:41:06,856 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T00:41:06,866 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T00:41:06,871 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:41:06,895 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4466 (auto-detected) 2024-12-05T00:41:06,896 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T00:41:06,914 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41705 2024-12-05T00:41:06,940 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41705 connecting to ZooKeeper ensemble=127.0.0.1:50354 2024-12-05T00:41:07,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:417050x0, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:41:07,072 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41705-0x101a2ef632c0000 connected 2024-12-05T00:41:07,371 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:07,373 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:07,381 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:41:07,385 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18, hbase.cluster.distributed=false 2024-12-05T00:41:07,410 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:41:07,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41705 2024-12-05T00:41:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41705 2024-12-05T00:41:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41705 2024-12-05T00:41:07,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41705 2024-12-05T00:41:07,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41705 2024-12-05T00:41:07,507 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fea72ea5c4b6:0 server-side Connection retries=45 2024-12-05T00:41:07,508 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:41:07,508 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T00:41:07,508 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T00:41:07,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T00:41:07,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T00:41:07,511 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T00:41:07,513 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T00:41:07,514 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42683 2024-12-05T00:41:07,515 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42683 connecting to ZooKeeper ensemble=127.0.0.1:50354 2024-12-05T00:41:07,517 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:07,519 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:07,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426830x0, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T00:41:07,585 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426830x0, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:41:07,586 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42683-0x101a2ef632c0001 connected 2024-12-05T00:41:07,591 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T00:41:07,599 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T00:41:07,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T00:41:07,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T00:41:07,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42683 2024-12-05T00:41:07,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42683 2024-12-05T00:41:07,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42683 2024-12-05T00:41:07,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42683 2024-12-05T00:41:07,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42683 2024-12-05T00:41:07,624 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fea72ea5c4b6:41705 2024-12-05T00:41:07,625 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:07,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:41:07,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:41:07,656 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:07,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T00:41:07,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:07,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:07,713 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T00:41:07,715 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fea72ea5c4b6,41705,1733359266486 from backup master directory 2024-12-05T00:41:07,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:41:07,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:07,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T00:41:07,738 WARN [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:41:07,740 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:07,742 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T00:41:07,744 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T00:41:07,796 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/hbase.id] with ID: 03d73038-d2a3-4039-b0c4-73d5bd1f77b7 2024-12-05T00:41:07,796 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/.tmp/hbase.id 2024-12-05T00:41:07,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741826_1002 (size=42) 2024-12-05T00:41:08,209 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/.tmp/hbase.id]:[hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/hbase.id] 2024-12-05T00:41:08,260 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T00:41:08,264 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T00:41:08,282 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-05T00:41:08,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:08,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:08,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741827_1003 (size=196) 2024-12-05T00:41:08,781 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:41:08,783 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T00:41:08,797 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:08,801 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:41:08,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741828_1004 (size=1189) 2024-12-05T00:41:09,253 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store 2024-12-05T00:41:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741829_1005 (size=34) 2024-12-05T00:41:09,676 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T00:41:09,679 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:09,680 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T00:41:09,681 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:41:09,681 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:41:09,683 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T00:41:09,683 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:41:09,683 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:41:09,685 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733359269680Disabling compacts and flushes for region at 1733359269680Disabling writes for close at 1733359269683 (+3 ms)Writing region close event to WAL at 1733359269683Closed at 1733359269683 2024-12-05T00:41:09,687 WARN [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/.initializing 2024-12-05T00:41:09,688 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/WALs/fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:09,695 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:41:09,709 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fea72ea5c4b6%2C41705%2C1733359266486, suffix=, logDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/WALs/fea72ea5c4b6,41705,1733359266486, archiveDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/oldWALs, maxLogs=10 2024-12-05T00:41:09,728 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/WALs/fea72ea5c4b6,41705,1733359266486/fea72ea5c4b6%2C41705%2C1733359266486.1733359269713, exclude list is [], retry=0 2024-12-05T00:41:09,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41735,DS-a5176426-ce12-425d-8b24-dcfe4287886a,DISK] 2024-12-05T00:41:09,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T00:41:09,784 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/WALs/fea72ea5c4b6,41705,1733359266486/fea72ea5c4b6%2C41705%2C1733359266486.1733359269713 2024-12-05T00:41:09,785 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42163:42163)] 2024-12-05T00:41:09,785 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:41:09,786 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:09,789 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,791 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,845 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T00:41:09,850 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:09,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:09,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,858 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T00:41:09,858 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:09,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:09,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T00:41:09,864 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:09,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:09,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T00:41:09,869 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:09,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:09,870 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,874 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,875 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,880 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,881 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,884 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T00:41:09,888 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T00:41:09,892 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:41:09,893 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63592300, jitterRate=-0.05240088701248169}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T00:41:09,897 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733359269802Initializing all the Stores at 1733359269804 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359269804Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359269805 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359269805Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359269805Cleaning up temporary data from old regions at 1733359269881 (+76 ms)Region opened successfully at 1733359269897 (+16 ms) 2024-12-05T00:41:09,898 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T00:41:09,927 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e88a09e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:41:09,953 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T00:41:09,962 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T00:41:09,962 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T00:41:09,965 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T00:41:09,966 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T00:41:09,970 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-05T00:41:09,970 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T00:41:09,992 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T00:41:10,000 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T00:41:10,011 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T00:41:10,013 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T00:41:10,015 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T00:41:10,032 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T00:41:10,035 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T00:41:10,038 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T00:41:10,051 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T00:41:10,053 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T00:41:10,074 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T00:41:10,093 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T00:41:10,116 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T00:41:10,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:41:10,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T00:41:10,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:10,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:10,129 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fea72ea5c4b6,41705,1733359266486, sessionid=0x101a2ef632c0000, setting cluster-up flag (Was=false) 2024-12-05T00:41:10,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:10,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:10,222 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T00:41:10,227 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:10,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:10,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:10,295 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T00:41:10,297 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:10,304 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T00:41:10,317 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(746): ClusterId : 03d73038-d2a3-4039-b0c4-73d5bd1f77b7 2024-12-05T00:41:10,320 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T00:41:10,330 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T00:41:10,330 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T00:41:10,339 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T00:41:10,340 DEBUG [RS:0;fea72ea5c4b6:42683 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae656aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fea72ea5c4b6/172.17.0.2:0 2024-12-05T00:41:10,359 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fea72ea5c4b6:42683 2024-12-05T00:41:10,362 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T00:41:10,362 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T00:41:10,362 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T00:41:10,364 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,41705,1733359266486 with port=42683, startcode=1733359267474 2024-12-05T00:41:10,375 DEBUG [RS:0;fea72ea5c4b6:42683 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T00:41:10,379 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T00:41:10,389 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T00:41:10,396 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T00:41:10,402 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fea72ea5c4b6,41705,1733359266486 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T00:41:10,407 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44447, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T00:41:10,410 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:41:10,410 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:41:10,410 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:41:10,410 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=5, maxPoolSize=5 2024-12-05T00:41:10,411 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fea72ea5c4b6:0, corePoolSize=10, maxPoolSize=10 2024-12-05T00:41:10,411 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,411 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:41:10,411 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,413 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733359300413 2024-12-05T00:41:10,414 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41705 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:10,415 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T00:41:10,416 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T00:41:10,417 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:41:10,418 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T00:41:10,419 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T00:41:10,420 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T00:41:10,420 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T00:41:10,420 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T00:41:10,424 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:10,423 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,425 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T00:41:10,428 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T00:41:10,430 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T00:41:10,430 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T00:41:10,432 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T00:41:10,432 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T00:41:10,435 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359270433,5,FailOnTimeoutGroup] 2024-12-05T00:41:10,436 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359270436,5,FailOnTimeoutGroup] 2024-12-05T00:41:10,436 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,437 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T00:41:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741831_1007 (size=1321) 2024-12-05T00:41:10,438 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,439 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,440 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T00:41:10,441 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T00:41:10,441 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:41:10,441 WARN [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T00:41:10,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741832_1008 (size=32) 2024-12-05T00:41:10,542 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(2659): reportForDuty to master=fea72ea5c4b6,41705,1733359266486 with port=42683, startcode=1733359267474 2024-12-05T00:41:10,545 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41705 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:10,548 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41705 {}] master.ServerManager(517): Registering regionserver=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:10,558 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:41:10,558 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42819 2024-12-05T00:41:10,559 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T00:41:10,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:41:10,580 DEBUG [RS:0;fea72ea5c4b6:42683 {}] zookeeper.ZKUtil(111): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:10,580 WARN [RS:0;fea72ea5c4b6:42683 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T00:41:10,580 INFO [RS:0;fea72ea5c4b6:42683 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:41:10,581 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:10,583 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fea72ea5c4b6,42683,1733359267474] 2024-12-05T00:41:10,615 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T00:41:10,637 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T00:41:10,646 INFO [RS:0;fea72ea5c4b6:42683 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T00:41:10,647 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,648 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T00:41:10,656 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T00:41:10,658 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,658 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,658 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,659 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,659 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,659 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,659 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=2, maxPoolSize=2 2024-12-05T00:41:10,660 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,660 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,660 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,660 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,661 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,661 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fea72ea5c4b6:0, corePoolSize=1, maxPoolSize=1 2024-12-05T00:41:10,661 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:41:10,661 DEBUG [RS:0;fea72ea5c4b6:42683 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0, corePoolSize=3, maxPoolSize=3 2024-12-05T00:41:10,663 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,663 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,663 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,664 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,664 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,664 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,42683,1733359267474-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:41:10,694 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T00:41:10,697 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,42683,1733359267474-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,698 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,698 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.Replication(171): fea72ea5c4b6,42683,1733359267474 started 2024-12-05T00:41:10,726 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:10,726 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1482): Serving as fea72ea5c4b6,42683,1733359267474, RpcServer on fea72ea5c4b6/172.17.0.2:42683, sessionid=0x101a2ef632c0001 2024-12-05T00:41:10,727 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T00:41:10,727 DEBUG [RS:0;fea72ea5c4b6:42683 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:10,728 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,42683,1733359267474' 2024-12-05T00:41:10,728 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T00:41:10,729 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T00:41:10,730 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T00:41:10,730 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T00:41:10,730 DEBUG [RS:0;fea72ea5c4b6:42683 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:10,730 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fea72ea5c4b6,42683,1733359267474' 2024-12-05T00:41:10,730 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T00:41:10,731 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T00:41:10,732 DEBUG [RS:0;fea72ea5c4b6:42683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T00:41:10,732 INFO [RS:0;fea72ea5c4b6:42683 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T00:41:10,732 INFO [RS:0;fea72ea5c4b6:42683 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T00:41:10,837 INFO [RS:0;fea72ea5c4b6:42683 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T00:41:10,840 INFO [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fea72ea5c4b6%2C42683%2C1733359267474, suffix=, logDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474, archiveDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/oldWALs, maxLogs=32 2024-12-05T00:41:10,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:10,856 DEBUG [RS:0;fea72ea5c4b6:42683 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474/fea72ea5c4b6%2C42683%2C1733359267474.1733359270842, exclude list is [], retry=0 2024-12-05T00:41:10,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T00:41:10,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41735,DS-a5176426-ce12-425d-8b24-dcfe4287886a,DISK] 2024-12-05T00:41:10,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T00:41:10,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:10,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:10,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T00:41:10,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T00:41:10,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:10,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:10,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T00:41:10,871 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T00:41:10,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:10,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:10,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T00:41:10,882 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T00:41:10,882 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:10,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:10,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T00:41:10,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740 2024-12-05T00:41:10,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740 2024-12-05T00:41:10,888 INFO [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474/fea72ea5c4b6%2C42683%2C1733359267474.1733359270842 2024-12-05T00:41:10,889 DEBUG [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42163:42163)] 2024-12-05T00:41:10,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T00:41:10,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T00:41:10,891 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:41:10,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T00:41:10,897 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:41:10,897 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73655448, jitterRate=0.09755170345306396}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:41:10,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733359270856Initializing all the Stores at 1733359270858 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359270858Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359270859 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359270859Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359270859Cleaning up temporary data from old regions at 1733359270890 (+31 ms)Region opened successfully at 1733359270899 (+9 ms) 2024-12-05T00:41:10,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T00:41:10,900 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T00:41:10,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T00:41:10,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T00:41:10,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T00:41:10,901 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T00:41:10,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733359270899Disabling compacts and flushes for region at 1733359270899Disabling writes for close at 1733359270900 (+1 ms)Writing region close event to WAL at 1733359270901 (+1 ms)Closed at 1733359270901 2024-12-05T00:41:10,904 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:41:10,904 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T00:41:10,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T00:41:10,917 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T00:41:10,920 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T00:41:11,072 DEBUG [fea72ea5c4b6:41705 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T00:41:11,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:11,090 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fea72ea5c4b6,42683,1733359267474, state=OPENING 2024-12-05T00:41:11,105 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T00:41:11,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:11,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:41:11,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:41:11,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:41:11,120 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T00:41:11,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:41:11,303 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T00:41:11,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56565, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T00:41:11,320 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T00:41:11,320 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T00:41:11,321 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T00:41:11,326 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fea72ea5c4b6%2C42683%2C1733359267474.meta, suffix=.meta, logDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474, archiveDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/oldWALs, maxLogs=32 2024-12-05T00:41:11,346 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474/fea72ea5c4b6%2C42683%2C1733359267474.meta.1733359271328.meta, exclude list is [], retry=0 2024-12-05T00:41:11,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41735,DS-a5176426-ce12-425d-8b24-dcfe4287886a,DISK] 2024-12-05T00:41:11,353 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/WALs/fea72ea5c4b6,42683,1733359267474/fea72ea5c4b6%2C42683%2C1733359267474.meta.1733359271328.meta 2024-12-05T00:41:11,353 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42163:42163)] 2024-12-05T00:41:11,354 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:41:11,355 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T00:41:11,358 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T00:41:11,362 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T00:41:11,366 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T00:41:11,367 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:11,367 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T00:41:11,367 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T00:41:11,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T00:41:11,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T00:41:11,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:11,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:11,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T00:41:11,375 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T00:41:11,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:11,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:11,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T00:41:11,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T00:41:11,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:11,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:11,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T00:41:11,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T00:41:11,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:11,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T00:41:11,381 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T00:41:11,383 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740 2024-12-05T00:41:11,386 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740 2024-12-05T00:41:11,388 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T00:41:11,388 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T00:41:11,389 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:41:11,392 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T00:41:11,394 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69229447, jitterRate=0.031599149107933044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:41:11,394 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T00:41:11,396 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733359271368Writing region info on filesystem at 1733359271368Initializing all the Stores at 1733359271370 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359271370Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359271370Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359271370Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733359271371 (+1 ms)Cleaning up temporary data from old regions at 1733359271389 (+18 ms)Running coprocessor post-open hooks at 1733359271394 (+5 ms)Region opened successfully at 1733359271396 (+2 ms) 2024-12-05T00:41:11,403 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733359271295 2024-12-05T00:41:11,415 DEBUG [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T00:41:11,415 INFO [RS_OPEN_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T00:41:11,416 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:11,419 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fea72ea5c4b6,42683,1733359267474, state=OPEN 2024-12-05T00:41:11,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:41:11,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T00:41:11,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:41:11,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T00:41:11,488 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:11,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T00:41:11,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fea72ea5c4b6,42683,1733359267474 in 366 msec 2024-12-05T00:41:11,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T00:41:11,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 590 msec 2024-12-05T00:41:11,509 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T00:41:11,509 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T00:41:11,528 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:11,530 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:11,548 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:11,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46671, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:11,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2310 sec 2024-12-05T00:41:11,569 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733359271569, completionTime=-1 2024-12-05T00:41:11,571 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T00:41:11,572 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T00:41:11,595 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T00:41:11,595 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733359331595 2024-12-05T00:41:11,595 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733359391595 2024-12-05T00:41:11,596 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-05T00:41:11,598 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,41705,1733359266486-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,598 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,41705,1733359266486-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,598 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,41705,1733359266486-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,600 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fea72ea5c4b6:41705, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,600 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,600 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,606 DEBUG [master/fea72ea5c4b6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T00:41:11,624 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.883sec 2024-12-05T00:41:11,625 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T00:41:11,625 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T00:41:11,626 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T00:41:11,626 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T00:41:11,627 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T00:41:11,627 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,41705,1733359266486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T00:41:11,628 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,41705,1733359266486-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T00:41:11,636 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T00:41:11,637 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T00:41:11,637 INFO [master/fea72ea5c4b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fea72ea5c4b6,41705,1733359266486-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T00:41:11,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c90ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:11,733 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T00:41:11,733 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T00:41:11,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:11,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:11,744 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72188503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:11,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:11,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba7d24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:11,758 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:11,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:11,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:11,796 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=10049 2024-12-05T00:41:11,804 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:41:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a30b8b7 2024-12-05T00:41:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:41:11,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:41:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:41:11,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:41:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:11,854 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:41:11,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-05T00:41:11,857 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:11,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:41:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:41:11,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741835_1011 (size=1061) 2024-12-05T00:41:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:41:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:41:12,280 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:41:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741836_1012 (size=53) 2024-12-05T00:41:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:41:12,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:12,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d5061eee39269f2ef93c654f6e1034a0, disabling compactions & flushes 2024-12-05T00:41:12,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:12,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:12,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. after waiting 0 ms 2024-12-05T00:41:12,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:12,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:12,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d5061eee39269f2ef93c654f6e1034a0: Waiting for close lock at 1733359272690Disabling compacts and flushes for region at 1733359272690Disabling writes for close at 1733359272691 (+1 ms)Writing region close event to WAL at 1733359272691Closed at 1733359272691 2024-12-05T00:41:12,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:41:12,698 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733359272694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359272694"}]},"ts":"1733359272694"} 2024-12-05T00:41:12,703 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:41:12,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:41:12,708 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359272706"}]},"ts":"1733359272706"} 2024-12-05T00:41:12,713 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T00:41:12,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, ASSIGN}] 2024-12-05T00:41:12,717 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, ASSIGN 2024-12-05T00:41:12,719 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=false 2024-12-05T00:41:12,872 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d5061eee39269f2ef93c654f6e1034a0, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:12,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, ASSIGN because future has completed 2024-12-05T00:41:12,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:41:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:41:13,047 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:13,048 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:41:13,049 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,049 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:13,049 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,049 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,052 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,056 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:13,056 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5061eee39269f2ef93c654f6e1034a0 columnFamilyName A 2024-12-05T00:41:13,056 DEBUG [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:13,057 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.HStore(327): Store=d5061eee39269f2ef93c654f6e1034a0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:13,058 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,060 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:13,060 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5061eee39269f2ef93c654f6e1034a0 columnFamilyName B 2024-12-05T00:41:13,060 DEBUG [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:13,061 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.HStore(327): Store=d5061eee39269f2ef93c654f6e1034a0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:13,062 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,064 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:13,064 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5061eee39269f2ef93c654f6e1034a0 columnFamilyName C 2024-12-05T00:41:13,064 DEBUG [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:13,065 INFO [StoreOpener-d5061eee39269f2ef93c654f6e1034a0-1 {}] regionserver.HStore(327): Store=d5061eee39269f2ef93c654f6e1034a0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:13,066 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:13,066 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,067 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,068 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,070 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,070 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,071 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:41:13,073 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,076 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:41:13,077 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d5061eee39269f2ef93c654f6e1034a0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64349634, jitterRate=-0.04111573100090027}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:41:13,077 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:13,078 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d5061eee39269f2ef93c654f6e1034a0: Running coprocessor pre-open hook at 1733359273049Writing region info on filesystem at 1733359273049Initializing all the Stores at 1733359273052 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359273052Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359273052Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359273052Cleaning up temporary data from old regions at 1733359273070 (+18 ms)Running coprocessor post-open hooks at 1733359273077 (+7 ms)Region opened successfully at 1733359273078 (+1 ms) 2024-12-05T00:41:13,079 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., pid=6, masterSystemTime=1733359273037 2024-12-05T00:41:13,083 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:13,083 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:13,083 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d5061eee39269f2ef93c654f6e1034a0, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:13,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:41:13,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T00:41:13,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 in 207 msec 2024-12-05T00:41:13,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T00:41:13,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, ASSIGN in 378 msec 2024-12-05T00:41:13,097 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:41:13,097 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359273097"}]},"ts":"1733359273097"} 2024-12-05T00:41:13,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T00:41:13,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:41:13,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2590 sec 2024-12-05T00:41:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T00:41:14,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:14,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,024 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,029 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,035 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:41:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-12-05T00:41:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:41:14,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:41:14,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff18d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bd06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,052 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af20e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,059 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6b748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,063 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,069 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a031a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,072 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a1730c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb793a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,078 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60658e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,082 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643444eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d403639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,085 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,087 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d66d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,091 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,093 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be0d766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:14,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:14,096 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71258907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:14,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cfd059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:14,107 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,107 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,108 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,108 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,109 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:14,109 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,114 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,114 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-05T00:41:14,118 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,119 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:14,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,121 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T00:41:14,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:14,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:41:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:14,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:41:14,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:14,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:14,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T00:41:14,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T00:41:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359334287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359274269, max timestamp=1733359274297Num uniques 859; 2024-12-05T00:41:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359334298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:41:14,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/f123214359b34570934d2a5b831bb9ef is 50, key is test_row_0/A:col10/1733359274191/Put/seqid=0 2024-12-05T00:41:14,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359274269, max timestamp=1733359274297Num uniques 859; 2024-12-05T00:41:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:41:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359274269, max timestamp=1733359274297Num uniques 859; 2024-12-05T00:41:14,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:41:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741837_1013 (size=14341) 2024-12-05T00:41:14,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/f123214359b34570934d2a5b831bb9ef 2024-12-05T00:41:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359334410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359334414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359334420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359334433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359334434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T00:41:14,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T00:41:14,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/039d6324df824020bdff715723f051d7 is 50, key is test_row_0/B:col10/1733359274191/Put/seqid=0 2024-12-05T00:41:14,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:14,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359334533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359334532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359334542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741838_1014 (size=12001) 2024-12-05T00:41:14,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/039d6324df824020bdff715723f051d7 2024-12-05T00:41:14,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/59d30bd0acf3447097de50e0cb4c509f is 50, key is test_row_0/C:col10/1733359274191/Put/seqid=0 2024-12-05T00:41:14,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741839_1015 (size=12001) 2024-12-05T00:41:14,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359334642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359334652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T00:41:14,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359334742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359334743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359334753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T00:41:14,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T00:41:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359334963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359334972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:14,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T00:41:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:14,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:15,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/59d30bd0acf3447097de50e0cb4c509f 2024-12-05T00:41:15,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/f123214359b34570934d2a5b831bb9ef as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/f123214359b34570934d2a5b831bb9ef 2024-12-05T00:41:15,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359335055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359335055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/f123214359b34570934d2a5b831bb9ef, entries=200, sequenceid=13, filesize=14.0 K 2024-12-05T00:41:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359335065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/039d6324df824020bdff715723f051d7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/039d6324df824020bdff715723f051d7 2024-12-05T00:41:15,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/039d6324df824020bdff715723f051d7, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T00:41:15,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/59d30bd0acf3447097de50e0cb4c509f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/59d30bd0acf3447097de50e0cb4c509f 2024-12-05T00:41:15,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/59d30bd0acf3447097de50e0cb4c509f, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T00:41:15,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for d5061eee39269f2ef93c654f6e1034a0 in 931ms, sequenceid=13, compaction requested=false 2024-12-05T00:41:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T00:41:15,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:15,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T00:41:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:15,146 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-05T00:41:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:15,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:15,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7b1ec73af6c1441fb4c8c0c11047f5e9 is 50, key is test_row_0/A:col10/1733359274412/Put/seqid=0 2024-12-05T00:41:15,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741840_1016 (size=12001) 2024-12-05T00:41:15,188 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7b1ec73af6c1441fb4c8c0c11047f5e9 2024-12-05T00:41:15,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/d397f22b376741bbb561b3818a6af104 is 50, key is test_row_0/B:col10/1733359274412/Put/seqid=0 2024-12-05T00:41:15,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741841_1017 (size=12001) 2024-12-05T00:41:15,241 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/d397f22b376741bbb561b3818a6af104 2024-12-05T00:41:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T00:41:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/64feba6352fb4478b726fd1017417702 is 50, key is test_row_0/C:col10/1733359274412/Put/seqid=0 2024-12-05T00:41:15,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741842_1018 (size=12001) 2024-12-05T00:41:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:15,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359335556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359335563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359335573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359335574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359335574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359335672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359335672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,675 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,675 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,675 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/64feba6352fb4478b726fd1017417702 2024-12-05T00:41:15,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7b1ec73af6c1441fb4c8c0c11047f5e9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7b1ec73af6c1441fb4c8c0c11047f5e9 2024-12-05T00:41:15,747 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7b1ec73af6c1441fb4c8c0c11047f5e9, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T00:41:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/d397f22b376741bbb561b3818a6af104 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/d397f22b376741bbb561b3818a6af104 2024-12-05T00:41:15,770 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/d397f22b376741bbb561b3818a6af104, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T00:41:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/64feba6352fb4478b726fd1017417702 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/64feba6352fb4478b726fd1017417702 2024-12-05T00:41:15,801 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/64feba6352fb4478b726fd1017417702, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T00:41:15,804 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for d5061eee39269f2ef93c654f6e1034a0 in 658ms, sequenceid=47, compaction requested=false 2024-12-05T00:41:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-05T00:41:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-05T00:41:15,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T00:41:15,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6720 sec 2024-12-05T00:41:15,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7030 sec 2024-12-05T00:41:15,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:41:15,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:15,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:15,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:15,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:15,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:15,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:15,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/686796c5de87421688d6d223f625a703 is 50, key is test_row_0/A:col10/1733359275526/Put/seqid=0 2024-12-05T00:41:15,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741843_1019 (size=14341) 2024-12-05T00:41:15,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/686796c5de87421688d6d223f625a703 2024-12-05T00:41:15,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/09f9e6b5218346aeab2be7c3c61ce010 is 50, key is test_row_0/B:col10/1733359275526/Put/seqid=0 2024-12-05T00:41:15,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359335962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359335968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:15,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741844_1020 (size=12001) 2024-12-05T00:41:15,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/09f9e6b5218346aeab2be7c3c61ce010 2024-12-05T00:41:16,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/25415846c2cc429c89abbdf60e263940 is 50, key is test_row_0/C:col10/1733359275526/Put/seqid=0 2024-12-05T00:41:16,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741845_1021 (size=12001) 2024-12-05T00:41:16,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359336071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359336082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359336282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T00:41:16,292 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:16,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359336292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-05T00:41:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,296 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:16,298 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:16,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:16,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/25415846c2cc429c89abbdf60e263940 2024-12-05T00:41:16,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/686796c5de87421688d6d223f625a703 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/686796c5de87421688d6d223f625a703 2024-12-05T00:41:16,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/686796c5de87421688d6d223f625a703, entries=200, sequenceid=60, filesize=14.0 K 2024-12-05T00:41:16,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:16,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:16,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/09f9e6b5218346aeab2be7c3c61ce010 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/09f9e6b5218346aeab2be7c3c61ce010 2024-12-05T00:41:16,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/09f9e6b5218346aeab2be7c3c61ce010, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T00:41:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/25415846c2cc429c89abbdf60e263940 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/25415846c2cc429c89abbdf60e263940 2024-12-05T00:41:16,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/25415846c2cc429c89abbdf60e263940, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T00:41:16,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d5061eee39269f2ef93c654f6e1034a0 in 623ms, sequenceid=60, compaction requested=true 2024-12-05T00:41:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:16,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:16,520 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:16,520 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:16,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:16,524 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:16,525 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:16,526 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,526 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/039d6324df824020bdff715723f051d7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/d397f22b376741bbb561b3818a6af104, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/09f9e6b5218346aeab2be7c3c61ce010] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=35.2 K 2024-12-05T00:41:16,528 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:16,528 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 039d6324df824020bdff715723f051d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733359274153 2024-12-05T00:41:16,529 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,529 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/f123214359b34570934d2a5b831bb9ef, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7b1ec73af6c1441fb4c8c0c11047f5e9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/686796c5de87421688d6d223f625a703] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=39.7 K 2024-12-05T00:41:16,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d397f22b376741bbb561b3818a6af104, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733359274401 2024-12-05T00:41:16,531 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 09f9e6b5218346aeab2be7c3c61ce010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733359275526 2024-12-05T00:41:16,531 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f123214359b34570934d2a5b831bb9ef, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733359274153 2024-12-05T00:41:16,532 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b1ec73af6c1441fb4c8c0c11047f5e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733359274401 2024-12-05T00:41:16,533 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686796c5de87421688d6d223f625a703, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733359275526 2024-12-05T00:41:16,570 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:16,571 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/141f605af57947fb8897fecdd4e42382 is 50, key is test_row_0/B:col10/1733359275526/Put/seqid=0 2024-12-05T00:41:16,577 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#10 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:16,578 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/ba56a08413634992b3b54e1821301304 is 50, key is test_row_0/A:col10/1733359275526/Put/seqid=0 2024-12-05T00:41:16,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741846_1022 (size=12104) 2024-12-05T00:41:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:16,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:41:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:16,608 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/141f605af57947fb8897fecdd4e42382 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/141f605af57947fb8897fecdd4e42382 2024-12-05T00:41:16,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:16,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:16,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/68b1e78106c041a8978d1703ecf0a728 is 50, key is test_row_0/A:col10/1733359275961/Put/seqid=0 2024-12-05T00:41:16,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741847_1023 (size=12104) 2024-12-05T00:41:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359336624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,647 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 141f605af57947fb8897fecdd4e42382(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:16,647 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359336623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359336627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359336632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,650 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/ba56a08413634992b3b54e1821301304 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ba56a08413634992b3b54e1821301304 2024-12-05T00:41:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,652 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359276520; duration=0sec 2024-12-05T00:41:16,652 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:16,652 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:16,653 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:16,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359336645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,656 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:16,657 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:16,657 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,657 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/59d30bd0acf3447097de50e0cb4c509f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/64feba6352fb4478b726fd1017417702, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/25415846c2cc429c89abbdf60e263940] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=35.2 K 2024-12-05T00:41:16,659 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d30bd0acf3447097de50e0cb4c509f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733359274153 2024-12-05T00:41:16,661 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 64feba6352fb4478b726fd1017417702, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733359274401 2024-12-05T00:41:16,662 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 25415846c2cc429c89abbdf60e263940, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733359275526 2024-12-05T00:41:16,666 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into ba56a08413634992b3b54e1821301304(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:16,666 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:16,666 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359276516; duration=0sec 2024-12-05T00:41:16,666 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:16,666 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:16,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741848_1024 (size=14341) 2024-12-05T00:41:16,692 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:16,693 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/4a130724a39146268f4c186c8f6e7c48 is 50, key is test_row_0/C:col10/1733359275526/Put/seqid=0 2024-12-05T00:41:16,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741849_1025 (size=12104) 2024-12-05T00:41:16,723 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/4a130724a39146268f4c186c8f6e7c48 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/4a130724a39146268f4c186c8f6e7c48 2024-12-05T00:41:16,739 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 4a130724a39146268f4c186c8f6e7c48(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:16,739 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:16,740 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359276520; duration=0sec 2024-12-05T00:41:16,740 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:16,740 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359336752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359336752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359336752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359336761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359336761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:16,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,878 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:41:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:16,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:16,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:16,945 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T00:41:16,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T00:41:16,948 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T00:41:16,950 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T00:41:16,951 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T00:41:16,951 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T00:41:16,951 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T00:41:16,953 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T00:41:16,953 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T00:41:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359336962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359336962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359336962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359336972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:16,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359336972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/68b1e78106c041a8978d1703ecf0a728 2024-12-05T00:41:17,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/03ba6ae7a0844dada3764c923804a44c is 50, key is test_row_0/B:col10/1733359275961/Put/seqid=0 2024-12-05T00:41:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741850_1026 (size=12001) 2024-12-05T00:41:17,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/03ba6ae7a0844dada3764c923804a44c 2024-12-05T00:41:17,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/aa8fc32a41b84795976f8b1cf7701d83 is 50, key is test_row_0/C:col10/1733359275961/Put/seqid=0 2024-12-05T00:41:17,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741851_1027 (size=12001) 2024-12-05T00:41:17,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/aa8fc32a41b84795976f8b1cf7701d83 2024-12-05T00:41:17,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/68b1e78106c041a8978d1703ecf0a728 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/68b1e78106c041a8978d1703ecf0a728 2024-12-05T00:41:17,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/68b1e78106c041a8978d1703ecf0a728, entries=200, sequenceid=80, filesize=14.0 K 2024-12-05T00:41:17,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/03ba6ae7a0844dada3764c923804a44c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/03ba6ae7a0844dada3764c923804a44c 2024-12-05T00:41:17,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/03ba6ae7a0844dada3764c923804a44c, entries=150, sequenceid=80, filesize=11.7 K 2024-12-05T00:41:17,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/aa8fc32a41b84795976f8b1cf7701d83 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/aa8fc32a41b84795976f8b1cf7701d83 2024-12-05T00:41:17,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/aa8fc32a41b84795976f8b1cf7701d83, entries=150, sequenceid=80, filesize=11.7 K 2024-12-05T00:41:17,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d5061eee39269f2ef93c654f6e1034a0 in 655ms, sequenceid=80, compaction requested=false 2024-12-05T00:41:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:17,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:41:17,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:17,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:17,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:17,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:17,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:17,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:17,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/3934453dc7264668aeba528b6bbfa829 is 50, key is test_row_0/A:col10/1733359277274/Put/seqid=0 2024-12-05T00:41:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741852_1028 (size=12001) 2024-12-05T00:41:17,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359337312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359337317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359337328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359337328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359337330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:17,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359337432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359337442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359337442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359337452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359337452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:17,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:17,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359337652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359337652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359337653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359337662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359337662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:17,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/3934453dc7264668aeba528b6bbfa829 2024-12-05T00:41:17,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a262d72339474f4fbe1b786c439abfc2 is 50, key is test_row_0/B:col10/1733359277274/Put/seqid=0 2024-12-05T00:41:17,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741853_1029 (size=12001) 2024-12-05T00:41:17,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a262d72339474f4fbe1b786c439abfc2 2024-12-05T00:41:17,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/8e8b06cb0fa14c56b52b3d21d68cb36e is 50, key is test_row_0/C:col10/1733359277274/Put/seqid=0 2024-12-05T00:41:17,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741854_1030 (size=12001) 2024-12-05T00:41:17,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/8e8b06cb0fa14c56b52b3d21d68cb36e 2024-12-05T00:41:17,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/3934453dc7264668aeba528b6bbfa829 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/3934453dc7264668aeba528b6bbfa829 2024-12-05T00:41:17,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/3934453dc7264668aeba528b6bbfa829, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:41:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a262d72339474f4fbe1b786c439abfc2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a262d72339474f4fbe1b786c439abfc2 2024-12-05T00:41:17,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:17,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a262d72339474f4fbe1b786c439abfc2, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:41:17,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/8e8b06cb0fa14c56b52b3d21d68cb36e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8e8b06cb0fa14c56b52b3d21d68cb36e 2024-12-05T00:41:17,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8e8b06cb0fa14c56b52b3d21d68cb36e, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:41:17,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d5061eee39269f2ef93c654f6e1034a0 in 608ms, sequenceid=97, compaction requested=true 2024-12-05T00:41:17,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:17,885 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:17,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:17,885 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:17,887 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:17,887 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:17,887 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:17,887 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:17,887 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,887 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,887 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ba56a08413634992b3b54e1821301304, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/68b1e78106c041a8978d1703ecf0a728, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/3934453dc7264668aeba528b6bbfa829] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=37.5 K 2024-12-05T00:41:17,887 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/141f605af57947fb8897fecdd4e42382, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/03ba6ae7a0844dada3764c923804a44c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a262d72339474f4fbe1b786c439abfc2] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=35.3 K 2024-12-05T00:41:17,888 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ba56a08413634992b3b54e1821301304, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733359275526 2024-12-05T00:41:17,888 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141f605af57947fb8897fecdd4e42382, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733359275526 2024-12-05T00:41:17,889 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ba6ae7a0844dada3764c923804a44c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733359275957 2024-12-05T00:41:17,890 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 68b1e78106c041a8978d1703ecf0a728, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733359275954 2024-12-05T00:41:17,890 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a262d72339474f4fbe1b786c439abfc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359276621 2024-12-05T00:41:17,891 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3934453dc7264668aeba528b6bbfa829, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359276621 2024-12-05T00:41:17,919 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#18 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:17,920 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/396fd56994464216a6c9d589425cc942 is 50, key is test_row_0/B:col10/1733359277274/Put/seqid=0 2024-12-05T00:41:17,922 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:17,923 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/2bb9732de585467f8688e79bc462c5c6 is 50, key is test_row_0/A:col10/1733359277274/Put/seqid=0 2024-12-05T00:41:17,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741855_1031 (size=12207) 2024-12-05T00:41:17,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741856_1032 (size=12207) 2024-12-05T00:41:17,965 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/2bb9732de585467f8688e79bc462c5c6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2bb9732de585467f8688e79bc462c5c6 2024-12-05T00:41:17,969 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/396fd56994464216a6c9d589425cc942 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/396fd56994464216a6c9d589425cc942 2024-12-05T00:41:17,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359277311, max timestamp=1733359277966Num uniques 859; 2024-12-05T00:41:17,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:41:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:17,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=93.93 KB heapSize=272.27 KB 2024-12-05T00:41:17,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359277311, max timestamp=1733359277966Num uniques 859; 2024-12-05T00:41:17,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:41:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:17,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/a005f11ea15d41c8ac7651331135bb5b is 50, key is test_row_0/A:col10/1733359277314/Put/seqid=0 2024-12-05T00:41:17,987 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into 2bb9732de585467f8688e79bc462c5c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:17,988 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:17,988 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359277884; duration=0sec 2024-12-05T00:41:17,988 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:17,988 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:17,988 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:17,991 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:17,991 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:17,991 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:17,991 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/4a130724a39146268f4c186c8f6e7c48, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/aa8fc32a41b84795976f8b1cf7701d83, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8e8b06cb0fa14c56b52b3d21d68cb36e] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=35.3 K 2024-12-05T00:41:17,992 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 396fd56994464216a6c9d589425cc942(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:17,992 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:17,992 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a130724a39146268f4c186c8f6e7c48, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733359275526 2024-12-05T00:41:17,992 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359277885; duration=0sec 2024-12-05T00:41:17,992 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:17,993 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting aa8fc32a41b84795976f8b1cf7701d83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733359275957 2024-12-05T00:41:17,994 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8b06cb0fa14c56b52b3d21d68cb36e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359276621 2024-12-05T00:41:17,994 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:18,010 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:18,010 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/46302e889d064f359f9ae17dd3de895c is 50, key is test_row_0/C:col10/1733359277274/Put/seqid=0 2024-12-05T00:41:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741857_1033 (size=12001) 2024-12-05T00:41:18,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741858_1034 (size=12207) 2024-12-05T00:41:18,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359338025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359338028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359338028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359338030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359338033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,045 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/46302e889d064f359f9ae17dd3de895c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/46302e889d064f359f9ae17dd3de895c 2024-12-05T00:41:18,061 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 46302e889d064f359f9ae17dd3de895c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:18,061 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:18,061 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359277885; duration=0sec 2024-12-05T00:41:18,062 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:18,063 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359338142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359338142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359338142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359338144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359338151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359338352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359338352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359338352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359338352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359338362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/a005f11ea15d41c8ac7651331135bb5b 2024-12-05T00:41:18,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a6f2c02ceb3a49fdb5c3ad56422fdea6 is 50, key is test_row_0/B:col10/1733359277314/Put/seqid=0 2024-12-05T00:41:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:18,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741859_1035 (size=12001) 2024-12-05T00:41:18,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a6f2c02ceb3a49fdb5c3ad56422fdea6 2024-12-05T00:41:18,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/a9732f4f058445c39d5d9b75a2fcc909 is 50, key is test_row_0/C:col10/1733359277314/Put/seqid=0 2024-12-05T00:41:18,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741860_1036 (size=12001) 2024-12-05T00:41:18,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359338661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359338661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359338662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359338662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359338672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:18,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:18,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/a9732f4f058445c39d5d9b75a2fcc909 2024-12-05T00:41:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/a005f11ea15d41c8ac7651331135bb5b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a005f11ea15d41c8ac7651331135bb5b 2024-12-05T00:41:18,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a005f11ea15d41c8ac7651331135bb5b, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T00:41:18,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a6f2c02ceb3a49fdb5c3ad56422fdea6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a6f2c02ceb3a49fdb5c3ad56422fdea6 2024-12-05T00:41:18,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a6f2c02ceb3a49fdb5c3ad56422fdea6, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T00:41:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/a9732f4f058445c39d5d9b75a2fcc909 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/a9732f4f058445c39d5d9b75a2fcc909 2024-12-05T00:41:18,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/a9732f4f058445c39d5d9b75a2fcc909, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T00:41:18,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~184.44 KB/188864, currentSize=114.05 KB/116790 for d5061eee39269f2ef93c654f6e1034a0 in 950ms, sequenceid=118, compaction requested=false 2024-12-05T00:41:18,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:18,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T00:41:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:18,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:41:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:18,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/04fa85f7fa3e4ad38371aa87283541eb is 50, key is test_row_0/A:col10/1733359278024/Put/seqid=0 2024-12-05T00:41:18,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741861_1037 (size=12151) 2024-12-05T00:41:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:19,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:19,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359339188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359339186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359339194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359339195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359339195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359339302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359339303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359339302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359339312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359339312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,360 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/04fa85f7fa3e4ad38371aa87283541eb 2024-12-05T00:41:19,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/eb7680dbf3384ae2ac7ce98e10aa2539 is 50, key is test_row_0/B:col10/1733359278024/Put/seqid=0 2024-12-05T00:41:19,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741862_1038 (size=12151) 2024-12-05T00:41:19,392 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/eb7680dbf3384ae2ac7ce98e10aa2539 2024-12-05T00:41:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/70b45467f2144fe1bd15b750778af596 is 50, key is test_row_0/C:col10/1733359278024/Put/seqid=0 2024-12-05T00:41:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741863_1039 (size=12151) 2024-12-05T00:41:19,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359339512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359339512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359339512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359339522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359339522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359339822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359339822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359339822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,827 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/70b45467f2144fe1bd15b750778af596 2024-12-05T00:41:19,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359339833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359339833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:19,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/04fa85f7fa3e4ad38371aa87283541eb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/04fa85f7fa3e4ad38371aa87283541eb 2024-12-05T00:41:19,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/04fa85f7fa3e4ad38371aa87283541eb, entries=150, sequenceid=141, filesize=11.9 K 2024-12-05T00:41:19,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/eb7680dbf3384ae2ac7ce98e10aa2539 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/eb7680dbf3384ae2ac7ce98e10aa2539 2024-12-05T00:41:19,863 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/eb7680dbf3384ae2ac7ce98e10aa2539, entries=150, sequenceid=141, filesize=11.9 K 2024-12-05T00:41:19,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/70b45467f2144fe1bd15b750778af596 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/70b45467f2144fe1bd15b750778af596 2024-12-05T00:41:19,879 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/70b45467f2144fe1bd15b750778af596, entries=150, sequenceid=141, filesize=11.9 K 2024-12-05T00:41:19,881 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for d5061eee39269f2ef93c654f6e1034a0 in 939ms, sequenceid=141, compaction requested=true 2024-12-05T00:41:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T00:41:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-05T00:41:19,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T00:41:19,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5850 sec 2024-12-05T00:41:19,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 3.5950 sec 2024-12-05T00:41:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:20,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T00:41:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:20,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/b3712c52f315416992d2ff7b36a64c31 is 50, key is test_row_0/A:col10/1733359279192/Put/seqid=0 2024-12-05T00:41:20,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359340363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359340363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359340364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359340366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359340366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741864_1040 (size=14541) 2024-12-05T00:41:20,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/b3712c52f315416992d2ff7b36a64c31 2024-12-05T00:41:20,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ddc11aca753c4f468779422941a8190d is 50, key is test_row_0/B:col10/1733359279192/Put/seqid=0 2024-12-05T00:41:20,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741865_1041 (size=12151) 2024-12-05T00:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T00:41:20,472 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:20,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:20,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359340472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359340472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-05T00:41:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359340472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T00:41:20,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:20,477 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:20,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359340482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359340482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T00:41:20,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T00:41:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:20,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359340682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359340682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359340682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359340691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359340692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T00:41:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:20,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T00:41:20,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ddc11aca753c4f468779422941a8190d 2024-12-05T00:41:20,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/026fc4592bc04e18b983b96c8ff7a115 is 50, key is test_row_0/C:col10/1733359279192/Put/seqid=0 2024-12-05T00:41:20,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741866_1042 (size=12151) 2024-12-05T00:41:20,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T00:41:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:20,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:20,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359340991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359340991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359340991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:20,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359341002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359341002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T00:41:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T00:41:21,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/026fc4592bc04e18b983b96c8ff7a115 2024-12-05T00:41:21,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T00:41:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:21,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/b3712c52f315416992d2ff7b36a64c31 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/b3712c52f315416992d2ff7b36a64c31 2024-12-05T00:41:21,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/b3712c52f315416992d2ff7b36a64c31, entries=200, sequenceid=157, filesize=14.2 K 2024-12-05T00:41:21,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ddc11aca753c4f468779422941a8190d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddc11aca753c4f468779422941a8190d 2024-12-05T00:41:21,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddc11aca753c4f468779422941a8190d, entries=150, sequenceid=157, filesize=11.9 K 2024-12-05T00:41:21,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/026fc4592bc04e18b983b96c8ff7a115 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/026fc4592bc04e18b983b96c8ff7a115 2024-12-05T00:41:21,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/026fc4592bc04e18b983b96c8ff7a115, entries=150, sequenceid=157, filesize=11.9 K 2024-12-05T00:41:21,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d5061eee39269f2ef93c654f6e1034a0 in 978ms, sequenceid=157, compaction requested=true 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:21,315 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:21,315 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:21,317 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50900 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:21,317 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:21,318 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,318 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2bb9732de585467f8688e79bc462c5c6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a005f11ea15d41c8ac7651331135bb5b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/04fa85f7fa3e4ad38371aa87283541eb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/b3712c52f315416992d2ff7b36a64c31] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=49.7 K 2024-12-05T00:41:21,318 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:21,318 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:21,319 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,319 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb9732de585467f8688e79bc462c5c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359276621 2024-12-05T00:41:21,319 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/396fd56994464216a6c9d589425cc942, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a6f2c02ceb3a49fdb5c3ad56422fdea6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/eb7680dbf3384ae2ac7ce98e10aa2539, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddc11aca753c4f468779422941a8190d] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=47.4 K 2024-12-05T00:41:21,320 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 396fd56994464216a6c9d589425cc942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359276621 2024-12-05T00:41:21,320 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a005f11ea15d41c8ac7651331135bb5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733359277314 2024-12-05T00:41:21,321 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f2c02ceb3a49fdb5c3ad56422fdea6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733359277314 2024-12-05T00:41:21,322 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04fa85f7fa3e4ad38371aa87283541eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733359278018 2024-12-05T00:41:21,322 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7680dbf3384ae2ac7ce98e10aa2539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733359278018 2024-12-05T00:41:21,323 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3712c52f315416992d2ff7b36a64c31, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733359279192 2024-12-05T00:41:21,324 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ddc11aca753c4f468779422941a8190d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733359279192 2024-12-05T00:41:21,349 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:21,349 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7827814211a34e95a79104f899c9cd85 is 50, key is test_row_0/A:col10/1733359279192/Put/seqid=0 2024-12-05T00:41:21,351 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:21,352 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/933808e545ea46aab2c3950989514639 is 50, key is test_row_0/B:col10/1733359279192/Put/seqid=0 2024-12-05T00:41:21,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741868_1044 (size=12493) 2024-12-05T00:41:21,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741867_1043 (size=12493) 2024-12-05T00:41:21,396 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/933808e545ea46aab2c3950989514639 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/933808e545ea46aab2c3950989514639 2024-12-05T00:41:21,400 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7827814211a34e95a79104f899c9cd85 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7827814211a34e95a79104f899c9cd85 2024-12-05T00:41:21,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T00:41:21,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T00:41:21,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:21,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:21,418 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 933808e545ea46aab2c3950989514639(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:21,418 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:21,418 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=12, startTime=1733359281315; duration=0sec 2024-12-05T00:41:21,418 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:21,418 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:21,418 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:21,419 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into 7827814211a34e95a79104f899c9cd85(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:21,419 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:21,419 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=12, startTime=1733359281315; duration=0sec 2024-12-05T00:41:21,419 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:21,420 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:21,422 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:21,422 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:21,422 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,423 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/46302e889d064f359f9ae17dd3de895c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/a9732f4f058445c39d5d9b75a2fcc909, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/70b45467f2144fe1bd15b750778af596, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/026fc4592bc04e18b983b96c8ff7a115] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=47.4 K 2024-12-05T00:41:21,423 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 46302e889d064f359f9ae17dd3de895c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359276621 2024-12-05T00:41:21,424 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a9732f4f058445c39d5d9b75a2fcc909, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733359277314 2024-12-05T00:41:21,425 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b45467f2144fe1bd15b750778af596, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733359278018 2024-12-05T00:41:21,425 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 026fc4592bc04e18b983b96c8ff7a115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733359279192 2024-12-05T00:41:21,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/a38de09ddd1042bd970bfebdc8ec2354 is 50, key is test_row_0/A:col10/1733359280363/Put/seqid=0 2024-12-05T00:41:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741869_1045 (size=12151) 2024-12-05T00:41:21,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/a38de09ddd1042bd970bfebdc8ec2354 2024-12-05T00:41:21,455 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#33 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:21,457 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/675ff977852544ac8f5f261b906257a5 is 50, key is test_row_0/C:col10/1733359279192/Put/seqid=0 2024-12-05T00:41:21,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741870_1046 (size=12493) 2024-12-05T00:41:21,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/50efc39f4f904adc9a70428e9a0607e0 is 50, key is test_row_0/B:col10/1733359280363/Put/seqid=0 2024-12-05T00:41:21,499 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/675ff977852544ac8f5f261b906257a5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/675ff977852544ac8f5f261b906257a5 2024-12-05T00:41:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:21,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:21,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741871_1047 (size=12151) 2024-12-05T00:41:21,516 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/50efc39f4f904adc9a70428e9a0607e0 2024-12-05T00:41:21,517 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 675ff977852544ac8f5f261b906257a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:21,517 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:21,517 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=12, startTime=1733359281315; duration=0sec 2024-12-05T00:41:21,517 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:21,517 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/165201ada7474c1eb369a84bb7c5f757 is 50, key is test_row_0/C:col10/1733359280363/Put/seqid=0 2024-12-05T00:41:21,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359341532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359341534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359341534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359341536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359341538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741872_1048 (size=12151) 2024-12-05T00:41:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T00:41:21,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359341641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359341652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359341652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359341652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359341653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359341852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359341861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359341861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359341862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359341862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:21,961 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/165201ada7474c1eb369a84bb7c5f757 2024-12-05T00:41:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/a38de09ddd1042bd970bfebdc8ec2354 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a38de09ddd1042bd970bfebdc8ec2354 2024-12-05T00:41:21,974 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a38de09ddd1042bd970bfebdc8ec2354, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T00:41:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/50efc39f4f904adc9a70428e9a0607e0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/50efc39f4f904adc9a70428e9a0607e0 2024-12-05T00:41:21,982 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/50efc39f4f904adc9a70428e9a0607e0, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T00:41:21,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/165201ada7474c1eb369a84bb7c5f757 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/165201ada7474c1eb369a84bb7c5f757 2024-12-05T00:41:21,991 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/165201ada7474c1eb369a84bb7c5f757, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T00:41:21,992 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d5061eee39269f2ef93c654f6e1034a0 in 587ms, sequenceid=173, compaction requested=false 2024-12-05T00:41:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T00:41:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-05T00:41:21,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T00:41:21,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-12-05T00:41:21,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5240 sec 2024-12-05T00:41:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:22,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T00:41:22,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:22,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/bb70f763c1aa407c8743c9936e072a57 is 50, key is test_row_0/A:col10/1733359282162/Put/seqid=0 2024-12-05T00:41:22,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359342184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359342186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359342187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359342189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359342191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741873_1049 (size=14541) 2024-12-05T00:41:22,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/bb70f763c1aa407c8743c9936e072a57 2024-12-05T00:41:22,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/2ecffd11cbfe442aafb1227d284cab70 is 50, key is test_row_0/B:col10/1733359282162/Put/seqid=0 2024-12-05T00:41:22,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741874_1050 (size=12151) 2024-12-05T00:41:22,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/2ecffd11cbfe442aafb1227d284cab70 2024-12-05T00:41:22,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/496632f2dcf341b1854c6acadf7861a1 is 50, key is test_row_0/C:col10/1733359282162/Put/seqid=0 2024-12-05T00:41:22,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359342303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359342303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359342303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359342303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359342303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741875_1051 (size=12151) 2024-12-05T00:41:22,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/496632f2dcf341b1854c6acadf7861a1 2024-12-05T00:41:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/bb70f763c1aa407c8743c9936e072a57 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/bb70f763c1aa407c8743c9936e072a57 2024-12-05T00:41:22,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/bb70f763c1aa407c8743c9936e072a57, entries=200, sequenceid=193, filesize=14.2 K 2024-12-05T00:41:22,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/2ecffd11cbfe442aafb1227d284cab70 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/2ecffd11cbfe442aafb1227d284cab70 2024-12-05T00:41:22,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/2ecffd11cbfe442aafb1227d284cab70, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T00:41:22,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/496632f2dcf341b1854c6acadf7861a1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/496632f2dcf341b1854c6acadf7861a1 2024-12-05T00:41:22,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/496632f2dcf341b1854c6acadf7861a1, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T00:41:22,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d5061eee39269f2ef93c654f6e1034a0 in 241ms, sequenceid=193, compaction requested=true 2024-12-05T00:41:22,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:22,406 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:22,406 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:22,407 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:22,407 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:22,408 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:22,408 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:22,408 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:22,408 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:22,408 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7827814211a34e95a79104f899c9cd85, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a38de09ddd1042bd970bfebdc8ec2354, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/bb70f763c1aa407c8743c9936e072a57] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=38.3 K 2024-12-05T00:41:22,409 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/933808e545ea46aab2c3950989514639, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/50efc39f4f904adc9a70428e9a0607e0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/2ecffd11cbfe442aafb1227d284cab70] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=35.9 K 2024-12-05T00:41:22,409 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7827814211a34e95a79104f899c9cd85, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733359279192 2024-12-05T00:41:22,409 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 933808e545ea46aab2c3950989514639, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733359279192 2024-12-05T00:41:22,410 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38de09ddd1042bd970bfebdc8ec2354, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733359280339 2024-12-05T00:41:22,410 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 50efc39f4f904adc9a70428e9a0607e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733359280339 2024-12-05T00:41:22,411 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb70f763c1aa407c8743c9936e072a57, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359281523 2024-12-05T00:41:22,412 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ecffd11cbfe442aafb1227d284cab70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359281523 2024-12-05T00:41:22,431 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:22,433 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7145eac3d0134432b6b32298ba1bc456 is 50, key is test_row_0/A:col10/1733359282162/Put/seqid=0 2024-12-05T00:41:22,438 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:22,439 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/5304a386f6284f908183a5c24b004b1d is 50, key is test_row_0/B:col10/1733359282162/Put/seqid=0 2024-12-05T00:41:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741876_1052 (size=12595) 2024-12-05T00:41:22,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741877_1053 (size=12595) 2024-12-05T00:41:22,510 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/5304a386f6284f908183a5c24b004b1d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5304a386f6284f908183a5c24b004b1d 2024-12-05T00:41:22,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T00:41:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:22,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:22,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:22,534 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 5304a386f6284f908183a5c24b004b1d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:22,534 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:22,534 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359282406; duration=0sec 2024-12-05T00:41:22,534 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:22,534 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:22,535 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:22,538 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:22,538 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:22,538 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:22,538 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/675ff977852544ac8f5f261b906257a5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/165201ada7474c1eb369a84bb7c5f757, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/496632f2dcf341b1854c6acadf7861a1] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=35.9 K 2024-12-05T00:41:22,539 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 675ff977852544ac8f5f261b906257a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733359279192 2024-12-05T00:41:22,540 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 165201ada7474c1eb369a84bb7c5f757, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733359280339 2024-12-05T00:41:22,540 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 496632f2dcf341b1854c6acadf7861a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359281523 2024-12-05T00:41:22,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d5101a97b68440d9ac01a827da2484a9 is 50, key is test_row_0/A:col10/1733359282172/Put/seqid=0 2024-12-05T00:41:22,559 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:22,559 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/8f8f31da7fa3475d8a413ae428d34faf is 50, key is test_row_0/C:col10/1733359282162/Put/seqid=0 2024-12-05T00:41:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359342552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359342556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359342558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359342561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359342562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741878_1054 (size=12151) 2024-12-05T00:41:22,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d5101a97b68440d9ac01a827da2484a9 2024-12-05T00:41:22,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/612bd9a695f34df593b05a7556d2e5e0 is 50, key is test_row_0/B:col10/1733359282172/Put/seqid=0 2024-12-05T00:41:22,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741879_1055 (size=12595) 2024-12-05T00:41:22,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741880_1056 (size=12151) 2024-12-05T00:41:22,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/612bd9a695f34df593b05a7556d2e5e0 2024-12-05T00:41:22,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/c5fd2711b9b64b1485a1ecdcd7ef7f61 is 50, key is test_row_0/C:col10/1733359282172/Put/seqid=0 2024-12-05T00:41:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T00:41:22,632 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:22,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-05T00:41:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T00:41:22,637 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:22,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:22,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741881_1057 (size=12151) 2024-12-05T00:41:22,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/c5fd2711b9b64b1485a1ecdcd7ef7f61 2024-12-05T00:41:22,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d5101a97b68440d9ac01a827da2484a9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d5101a97b68440d9ac01a827da2484a9 2024-12-05T00:41:22,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d5101a97b68440d9ac01a827da2484a9, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T00:41:22,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/612bd9a695f34df593b05a7556d2e5e0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/612bd9a695f34df593b05a7556d2e5e0 2024-12-05T00:41:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359342672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359342672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359342673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359342673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359342673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/612bd9a695f34df593b05a7556d2e5e0, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T00:41:22,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/c5fd2711b9b64b1485a1ecdcd7ef7f61 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5fd2711b9b64b1485a1ecdcd7ef7f61 2024-12-05T00:41:22,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5fd2711b9b64b1485a1ecdcd7ef7f61, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T00:41:22,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d5061eee39269f2ef93c654f6e1034a0 in 194ms, sequenceid=210, compaction requested=false 2024-12-05T00:41:22,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T00:41:22,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T00:41:22,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:22,795 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T00:41:22,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:22,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:22,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:22,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:22,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/0c300fd651a047bdb23358991bfc7dc5 is 50, key is test_row_0/A:col10/1733359282543/Put/seqid=0 2024-12-05T00:41:22,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741882_1058 (size=12151) 2024-12-05T00:41:22,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:22,898 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7145eac3d0134432b6b32298ba1bc456 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7145eac3d0134432b6b32298ba1bc456 2024-12-05T00:41:22,909 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into 7145eac3d0134432b6b32298ba1bc456(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:22,909 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:22,909 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359282406; duration=0sec 2024-12-05T00:41:22,909 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:22,909 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359342908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359342910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359342912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359342916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359342920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T00:41:23,010 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/8f8f31da7fa3475d8a413ae428d34faf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8f8f31da7fa3475d8a413ae428d34faf 2024-12-05T00:41:23,017 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 8f8f31da7fa3475d8a413ae428d34faf(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:23,017 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:23,017 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359282406; duration=0sec 2024-12-05T00:41:23,018 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:23,018 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359343022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359343022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359343031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359343031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359343032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/0c300fd651a047bdb23358991bfc7dc5 2024-12-05T00:41:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/21a82fb576a14e91a17f2a6e0eb29bfb is 50, key is test_row_0/B:col10/1733359282543/Put/seqid=0 2024-12-05T00:41:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741883_1059 (size=12151) 2024-12-05T00:41:23,228 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/21a82fb576a14e91a17f2a6e0eb29bfb 2024-12-05T00:41:23,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359343232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359343237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/6b4c32dc2170498c810f5fbd2c223b8e is 50, key is test_row_0/C:col10/1733359282543/Put/seqid=0 2024-12-05T00:41:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359343242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359343242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359343242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T00:41:23,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741884_1060 (size=12151) 2024-12-05T00:41:23,266 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/6b4c32dc2170498c810f5fbd2c223b8e 2024-12-05T00:41:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/0c300fd651a047bdb23358991bfc7dc5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/0c300fd651a047bdb23358991bfc7dc5 2024-12-05T00:41:23,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/0c300fd651a047bdb23358991bfc7dc5, entries=150, sequenceid=226, filesize=11.9 K 2024-12-05T00:41:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/21a82fb576a14e91a17f2a6e0eb29bfb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/21a82fb576a14e91a17f2a6e0eb29bfb 2024-12-05T00:41:23,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/21a82fb576a14e91a17f2a6e0eb29bfb, entries=150, sequenceid=226, filesize=11.9 K 2024-12-05T00:41:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/6b4c32dc2170498c810f5fbd2c223b8e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/6b4c32dc2170498c810f5fbd2c223b8e 2024-12-05T00:41:23,304 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/6b4c32dc2170498c810f5fbd2c223b8e, entries=150, sequenceid=226, filesize=11.9 K 2024-12-05T00:41:23,305 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d5061eee39269f2ef93c654f6e1034a0 in 510ms, sequenceid=226, compaction requested=true 2024-12-05T00:41:23,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:23,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T00:41:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-05T00:41:23,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T00:41:23,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 669 msec 2024-12-05T00:41:23,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 681 msec 2024-12-05T00:41:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:23,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T00:41:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:23,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/954efd8e901d4baea4dbee63d749d269 is 50, key is test_row_0/A:col10/1733359282914/Put/seqid=0 2024-12-05T00:41:23,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741885_1061 (size=14541) 2024-12-05T00:41:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359343561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359343563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359343563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359343565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359343565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359343672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359343672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359343672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359343672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359343672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T00:41:23,772 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:23,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-05T00:41:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T00:41:23,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:23,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:23,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359343881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T00:41:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359343881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359343882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359343882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359343882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:23,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T00:41:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:23,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:23,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/954efd8e901d4baea4dbee63d749d269 2024-12-05T00:41:23,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/375148a0a82d4516a616ae45074a7542 is 50, key is test_row_0/B:col10/1733359282914/Put/seqid=0 2024-12-05T00:41:23,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741886_1062 (size=12151) 2024-12-05T00:41:23,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/375148a0a82d4516a616ae45074a7542 2024-12-05T00:41:24,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/ba747b00f0f043bb8a2768528beeb89a is 50, key is test_row_0/C:col10/1733359282914/Put/seqid=0 2024-12-05T00:41:24,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741887_1063 (size=12151) 2024-12-05T00:41:24,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T00:41:24,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T00:41:24,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359344193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359344193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359344193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359344193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359344202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T00:41:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T00:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T00:41:24,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/ba747b00f0f043bb8a2768528beeb89a 2024-12-05T00:41:24,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/954efd8e901d4baea4dbee63d749d269 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/954efd8e901d4baea4dbee63d749d269 2024-12-05T00:41:24,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/954efd8e901d4baea4dbee63d749d269, entries=200, sequenceid=246, filesize=14.2 K 2024-12-05T00:41:24,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/375148a0a82d4516a616ae45074a7542 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/375148a0a82d4516a616ae45074a7542 2024-12-05T00:41:24,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/375148a0a82d4516a616ae45074a7542, entries=150, sequenceid=246, filesize=11.9 K 2024-12-05T00:41:24,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/ba747b00f0f043bb8a2768528beeb89a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/ba747b00f0f043bb8a2768528beeb89a 2024-12-05T00:41:24,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/ba747b00f0f043bb8a2768528beeb89a, entries=150, sequenceid=246, filesize=11.9 K 2024-12-05T00:41:24,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d5061eee39269f2ef93c654f6e1034a0 in 930ms, sequenceid=246, compaction requested=true 2024-12-05T00:41:24,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:24,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:24,475 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:24,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:24,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:24,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:24,475 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:24,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:24,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:24,477 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:24,477 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:24,477 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,477 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7145eac3d0134432b6b32298ba1bc456, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d5101a97b68440d9ac01a827da2484a9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/0c300fd651a047bdb23358991bfc7dc5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/954efd8e901d4baea4dbee63d749d269] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=50.2 K 2024-12-05T00:41:24,478 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:24,478 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:24,478 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,478 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5304a386f6284f908183a5c24b004b1d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/612bd9a695f34df593b05a7556d2e5e0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/21a82fb576a14e91a17f2a6e0eb29bfb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/375148a0a82d4516a616ae45074a7542] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=47.9 K 2024-12-05T00:41:24,479 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7145eac3d0134432b6b32298ba1bc456, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359281523 2024-12-05T00:41:24,479 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5101a97b68440d9ac01a827da2484a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733359282172 2024-12-05T00:41:24,479 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5304a386f6284f908183a5c24b004b1d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359281523 2024-12-05T00:41:24,480 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c300fd651a047bdb23358991bfc7dc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733359282543 2024-12-05T00:41:24,480 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 612bd9a695f34df593b05a7556d2e5e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733359282172 2024-12-05T00:41:24,480 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a82fb576a14e91a17f2a6e0eb29bfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733359282543 2024-12-05T00:41:24,481 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 375148a0a82d4516a616ae45074a7542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733359282909 2024-12-05T00:41:24,481 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954efd8e901d4baea4dbee63d749d269, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733359282909 2024-12-05T00:41:24,499 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:24,499 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/5cf05883b7d24982baddc8f16b38fa5c is 50, key is test_row_0/B:col10/1733359282914/Put/seqid=0 2024-12-05T00:41:24,507 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:24,507 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d206981b59e7487da24c7afa4df6e034 is 50, key is test_row_0/A:col10/1733359282914/Put/seqid=0 2024-12-05T00:41:24,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741888_1064 (size=12731) 2024-12-05T00:41:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741889_1065 (size=12731) 2024-12-05T00:41:24,541 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/5cf05883b7d24982baddc8f16b38fa5c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5cf05883b7d24982baddc8f16b38fa5c 2024-12-05T00:41:24,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:24,554 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 5cf05883b7d24982baddc8f16b38fa5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:24,554 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:24,554 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=12, startTime=1733359284475; duration=0sec 2024-12-05T00:41:24,554 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:24,554 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:24,555 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:24,557 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:24,557 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:24,558 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:24,558 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8f8f31da7fa3475d8a413ae428d34faf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5fd2711b9b64b1485a1ecdcd7ef7f61, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/6b4c32dc2170498c810f5fbd2c223b8e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/ba747b00f0f043bb8a2768528beeb89a] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=47.9 K 2024-12-05T00:41:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/1a43a63369d44e339ec2050fe9c4afb8 is 50, key is test_row_0/A:col10/1733359283555/Put/seqid=0 2024-12-05T00:41:24,558 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8f31da7fa3475d8a413ae428d34faf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359281523 2024-12-05T00:41:24,559 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c5fd2711b9b64b1485a1ecdcd7ef7f61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733359282172 2024-12-05T00:41:24,559 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b4c32dc2170498c810f5fbd2c223b8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733359282543 2024-12-05T00:41:24,560 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ba747b00f0f043bb8a2768528beeb89a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733359282909 2024-12-05T00:41:24,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741890_1066 (size=12251) 2024-12-05T00:41:24,576 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/1a43a63369d44e339ec2050fe9c4afb8 2024-12-05T00:41:24,583 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:24,583 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/c5bc6ab142e74d1583973a1927c73b79 is 50, key is test_row_0/C:col10/1733359282914/Put/seqid=0 2024-12-05T00:41:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/6829e1c00da346fc89cd76f3cb30d212 is 50, key is test_row_0/B:col10/1733359283555/Put/seqid=0 2024-12-05T00:41:24,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741891_1067 (size=12731) 2024-12-05T00:41:24,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741892_1068 (size=12251) 2024-12-05T00:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:24,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359344721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359344721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359344723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359344724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359344725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359344831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359344831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359344842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359344842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359344842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T00:41:24,933 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d206981b59e7487da24c7afa4df6e034 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d206981b59e7487da24c7afa4df6e034 2024-12-05T00:41:24,944 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into d206981b59e7487da24c7afa4df6e034(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:24,944 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:24,944 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=12, startTime=1733359284474; duration=0sec 2024-12-05T00:41:24,944 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:24,944 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:25,002 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/6829e1c00da346fc89cd76f3cb30d212 2024-12-05T00:41:25,007 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/c5bc6ab142e74d1583973a1927c73b79 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5bc6ab142e74d1583973a1927c73b79 2024-12-05T00:41:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/9ba571c9349b453c9631c71b74de1aef is 50, key is test_row_0/C:col10/1733359283555/Put/seqid=0 2024-12-05T00:41:25,018 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into c5bc6ab142e74d1583973a1927c73b79(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:25,018 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:25,018 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=12, startTime=1733359284475; duration=0sec 2024-12-05T00:41:25,018 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:25,018 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:25,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741893_1069 (size=12251) 2024-12-05T00:41:25,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359345042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359345042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359345051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359345051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359345051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359345351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359345351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359345362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359345362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359345362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,424 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/9ba571c9349b453c9631c71b74de1aef 2024-12-05T00:41:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/1a43a63369d44e339ec2050fe9c4afb8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1a43a63369d44e339ec2050fe9c4afb8 2024-12-05T00:41:25,439 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1a43a63369d44e339ec2050fe9c4afb8, entries=150, sequenceid=260, filesize=12.0 K 2024-12-05T00:41:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/6829e1c00da346fc89cd76f3cb30d212 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/6829e1c00da346fc89cd76f3cb30d212 2024-12-05T00:41:25,447 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/6829e1c00da346fc89cd76f3cb30d212, entries=150, sequenceid=260, filesize=12.0 K 2024-12-05T00:41:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/9ba571c9349b453c9631c71b74de1aef as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9ba571c9349b453c9631c71b74de1aef 2024-12-05T00:41:25,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9ba571c9349b453c9631c71b74de1aef, entries=150, sequenceid=260, filesize=12.0 K 2024-12-05T00:41:25,457 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d5061eee39269f2ef93c654f6e1034a0 in 910ms, sequenceid=260, compaction requested=false 2024-12-05T00:41:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T00:41:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-05T00:41:25,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T00:41:25,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6830 sec 2024-12-05T00:41:25,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6890 sec 2024-12-05T00:41:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:25,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:41:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:25,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/c94af1b5ce2845ffaeebd0500a2abe8c is 50, key is test_row_0/A:col10/1733359285864/Put/seqid=0 2024-12-05T00:41:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359345884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359345885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359345886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359345886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359345888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741894_1070 (size=12301) 2024-12-05T00:41:25,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/c94af1b5ce2845ffaeebd0500a2abe8c 2024-12-05T00:41:25,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/77487a49d9074bf383dd83a270179d6d is 50, key is test_row_0/B:col10/1733359285864/Put/seqid=0 2024-12-05T00:41:25,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741895_1071 (size=12301) 2024-12-05T00:41:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T00:41:25,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:25,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-05T00:41:25,925 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T00:41:25,926 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:25,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359345992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359345992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359345992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359345993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359346002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T00:41:26,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T00:41:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359346202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359346202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359346202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359346202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359346211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T00:41:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T00:41:26,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/77487a49d9074bf383dd83a270179d6d 2024-12-05T00:41:26,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/332a5157cc0845be82c4f7a1581b8ec4 is 50, key is test_row_0/C:col10/1733359285864/Put/seqid=0 2024-12-05T00:41:26,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741896_1072 (size=12301) 2024-12-05T00:41:26,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/332a5157cc0845be82c4f7a1581b8ec4 2024-12-05T00:41:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/c94af1b5ce2845ffaeebd0500a2abe8c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/c94af1b5ce2845ffaeebd0500a2abe8c 2024-12-05T00:41:26,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/c94af1b5ce2845ffaeebd0500a2abe8c, entries=150, sequenceid=283, filesize=12.0 K 2024-12-05T00:41:26,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/77487a49d9074bf383dd83a270179d6d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77487a49d9074bf383dd83a270179d6d 2024-12-05T00:41:26,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77487a49d9074bf383dd83a270179d6d, entries=150, sequenceid=283, filesize=12.0 K 2024-12-05T00:41:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/332a5157cc0845be82c4f7a1581b8ec4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/332a5157cc0845be82c4f7a1581b8ec4 2024-12-05T00:41:26,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/332a5157cc0845be82c4f7a1581b8ec4, entries=150, sequenceid=283, filesize=12.0 K 2024-12-05T00:41:26,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d5061eee39269f2ef93c654f6e1034a0 in 509ms, sequenceid=283, compaction requested=true 2024-12-05T00:41:26,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:26,377 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:26,377 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:26,378 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:26,378 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:26,378 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,378 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5cf05883b7d24982baddc8f16b38fa5c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/6829e1c00da346fc89cd76f3cb30d212, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77487a49d9074bf383dd83a270179d6d] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.4 K 2024-12-05T00:41:26,378 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:26,378 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:26,379 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,379 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d206981b59e7487da24c7afa4df6e034, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1a43a63369d44e339ec2050fe9c4afb8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/c94af1b5ce2845ffaeebd0500a2abe8c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.4 K 2024-12-05T00:41:26,379 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf05883b7d24982baddc8f16b38fa5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733359282909 2024-12-05T00:41:26,379 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d206981b59e7487da24c7afa4df6e034, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733359282909 2024-12-05T00:41:26,380 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6829e1c00da346fc89cd76f3cb30d212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733359283555 2024-12-05T00:41:26,380 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a43a63369d44e339ec2050fe9c4afb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733359283555 2024-12-05T00:41:26,380 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94af1b5ce2845ffaeebd0500a2abe8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733359284723 2024-12-05T00:41:26,380 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 77487a49d9074bf383dd83a270179d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733359284723 2024-12-05T00:41:26,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T00:41:26,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:26,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:26,394 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:26,394 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7263be73e2fd47bfb4593eff02ed402f is 50, key is test_row_0/A:col10/1733359285864/Put/seqid=0 2024-12-05T00:41:26,397 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/660556d2cc724396b0601d52d989ff51 is 50, key is test_row_0/A:col10/1733359285884/Put/seqid=0 2024-12-05T00:41:26,397 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/664c230becca4d1586ca3a90ff79f4bd is 50, key is test_row_0/B:col10/1733359285864/Put/seqid=0 2024-12-05T00:41:26,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741898_1074 (size=12301) 2024-12-05T00:41:26,402 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/660556d2cc724396b0601d52d989ff51 2024-12-05T00:41:26,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741899_1075 (size=12983) 2024-12-05T00:41:26,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741897_1073 (size=12983) 2024-12-05T00:41:26,420 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/664c230becca4d1586ca3a90ff79f4bd as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/664c230becca4d1586ca3a90ff79f4bd 2024-12-05T00:41:26,420 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/7263be73e2fd47bfb4593eff02ed402f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7263be73e2fd47bfb4593eff02ed402f 2024-12-05T00:41:26,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a2c788bcd52e43739b14eb95228f92a3 is 50, key is test_row_0/B:col10/1733359285884/Put/seqid=0 2024-12-05T00:41:26,430 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 664c230becca4d1586ca3a90ff79f4bd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:26,430 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:26,430 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359286377; duration=0sec 2024-12-05T00:41:26,430 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:26,430 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:26,431 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:26,432 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:26,432 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:26,432 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,432 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5bc6ab142e74d1583973a1927c73b79, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9ba571c9349b453c9631c71b74de1aef, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/332a5157cc0845be82c4f7a1581b8ec4] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.4 K 2024-12-05T00:41:26,433 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bc6ab142e74d1583973a1927c73b79, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733359282909 2024-12-05T00:41:26,433 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into 7263be73e2fd47bfb4593eff02ed402f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:26,433 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:26,433 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359286376; duration=0sec 2024-12-05T00:41:26,434 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:26,434 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:26,434 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba571c9349b453c9631c71b74de1aef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733359283555 2024-12-05T00:41:26,434 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 332a5157cc0845be82c4f7a1581b8ec4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733359284723 2024-12-05T00:41:26,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741900_1076 (size=12301) 2024-12-05T00:41:26,439 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a2c788bcd52e43739b14eb95228f92a3 2024-12-05T00:41:26,449 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:26,449 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/b2108f2fe9e643beb895636bc9684653 is 50, key is test_row_0/C:col10/1733359285864/Put/seqid=0 2024-12-05T00:41:26,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741901_1077 (size=12983) 2024-12-05T00:41:26,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/560698b06aef4e5184a76f5fcaaac50d is 50, key is test_row_0/C:col10/1733359285884/Put/seqid=0 2024-12-05T00:41:26,505 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/b2108f2fe9e643beb895636bc9684653 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b2108f2fe9e643beb895636bc9684653 2024-12-05T00:41:26,514 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into b2108f2fe9e643beb895636bc9684653(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:26,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:26,514 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359286377; duration=0sec 2024-12-05T00:41:26,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:26,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:26,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:26,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741902_1078 (size=12301) 2024-12-05T00:41:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359346529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359346530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359346532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359346533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359346533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T00:41:26,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359346642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359346642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359346642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359346642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359346643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359346852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359346852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359346852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359346852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359346852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:26,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/560698b06aef4e5184a76f5fcaaac50d 2024-12-05T00:41:26,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/660556d2cc724396b0601d52d989ff51 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/660556d2cc724396b0601d52d989ff51 2024-12-05T00:41:26,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/660556d2cc724396b0601d52d989ff51, entries=150, sequenceid=295, filesize=12.0 K 2024-12-05T00:41:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a2c788bcd52e43739b14eb95228f92a3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a2c788bcd52e43739b14eb95228f92a3 2024-12-05T00:41:26,939 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a2c788bcd52e43739b14eb95228f92a3, entries=150, sequenceid=295, filesize=12.0 K 2024-12-05T00:41:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/560698b06aef4e5184a76f5fcaaac50d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/560698b06aef4e5184a76f5fcaaac50d 2024-12-05T00:41:26,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/560698b06aef4e5184a76f5fcaaac50d, entries=150, sequenceid=295, filesize=12.0 K 2024-12-05T00:41:26,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d5061eee39269f2ef93c654f6e1034a0 in 560ms, sequenceid=295, compaction requested=false 2024-12-05T00:41:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T00:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-05T00:41:26,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T00:41:26,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0240 sec 2024-12-05T00:41:26,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0310 sec 2024-12-05T00:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T00:41:27,062 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:27,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-05T00:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T00:41:27,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:27,070 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:27,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:27,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359286527, max timestamp=1733359287162Num uniques 859; 2024-12-05T00:41:27,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:27,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359286527, max timestamp=1733359287162Num uniques 859; 2024-12-05T00:41:27,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:27,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359286527, max timestamp=1733359287162Num uniques 859; 2024-12-05T00:41:27,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:27,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-05T00:41:27,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T00:41:27,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/1b847741fa894a0b9a356c25c3607a01 is 50, key is test_row_0/A:col10/1733359287165/Put/seqid=0 2024-12-05T00:41:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741903_1079 (size=12301) 2024-12-05T00:41:27,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/1b847741fa894a0b9a356c25c3607a01 2024-12-05T00:41:27,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/53b6dd59f77f45a7ac681f5ba6c673f1 is 50, key is test_row_0/B:col10/1733359287165/Put/seqid=0 2024-12-05T00:41:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741904_1080 (size=12301) 2024-12-05T00:41:27,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359287196, max timestamp=1733359287208Num uniques 859; 2024-12-05T00:41:27,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359287196, max timestamp=1733359287208Num uniques 859; 2024-12-05T00:41:27,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:27,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:27,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/53b6dd59f77f45a7ac681f5ba6c673f1 2024-12-05T00:41:27,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733359287196, max timestamp=1733359287206Num uniques 859; 2024-12-05T00:41:27,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T00:41:27,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:27,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:27,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/13ef184fda124cea9c5c29e2f1c6e512 is 50, key is test_row_0/C:col10/1733359287165/Put/seqid=0 2024-12-05T00:41:27,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741905_1081 (size=12301) 2024-12-05T00:41:27,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/13ef184fda124cea9c5c29e2f1c6e512 2024-12-05T00:41:27,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/1b847741fa894a0b9a356c25c3607a01 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1b847741fa894a0b9a356c25c3607a01 2024-12-05T00:41:27,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1b847741fa894a0b9a356c25c3607a01, entries=150, sequenceid=323, filesize=12.0 K 2024-12-05T00:41:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/53b6dd59f77f45a7ac681f5ba6c673f1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/53b6dd59f77f45a7ac681f5ba6c673f1 2024-12-05T00:41:27,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/53b6dd59f77f45a7ac681f5ba6c673f1, entries=150, sequenceid=323, filesize=12.0 K 2024-12-05T00:41:27,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/13ef184fda124cea9c5c29e2f1c6e512 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/13ef184fda124cea9c5c29e2f1c6e512 2024-12-05T00:41:27,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/13ef184fda124cea9c5c29e2f1c6e512, entries=150, sequenceid=323, filesize=12.0 K 2024-12-05T00:41:27,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=113.66 KB/116386 for d5061eee39269f2ef93c654f6e1034a0 in 137ms, sequenceid=323, compaction requested=true 2024-12-05T00:41:27,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:27,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:27,305 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:27,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:27,305 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:27,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:27,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:27,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:27,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=133.79 KB heapSize=384.07 KB 2024-12-05T00:41:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:27,308 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:27,308 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:27,308 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:27,308 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:27,308 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,308 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,308 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/664c230becca4d1586ca3a90ff79f4bd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a2c788bcd52e43739b14eb95228f92a3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/53b6dd59f77f45a7ac681f5ba6c673f1] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.7 K 2024-12-05T00:41:27,308 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7263be73e2fd47bfb4593eff02ed402f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/660556d2cc724396b0601d52d989ff51, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1b847741fa894a0b9a356c25c3607a01] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.7 K 2024-12-05T00:41:27,309 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 664c230becca4d1586ca3a90ff79f4bd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733359284723 2024-12-05T00:41:27,309 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7263be73e2fd47bfb4593eff02ed402f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733359284723 2024-12-05T00:41:27,309 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c788bcd52e43739b14eb95228f92a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733359285882 2024-12-05T00:41:27,310 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 660556d2cc724396b0601d52d989ff51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733359285882 2024-12-05T00:41:27,310 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b6dd59f77f45a7ac681f5ba6c673f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733359287162 2024-12-05T00:41:27,310 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b847741fa894a0b9a356c25c3607a01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733359287162 2024-12-05T00:41:27,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/db0f3254913e4b839496e20aedf34599 is 50, key is test_row_0/A:col10/1733359287239/Put/seqid=0 2024-12-05T00:41:27,322 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:27,322 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:27,323 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/822e39df63b04414b7b463dbd4fa405b is 50, key is test_row_0/B:col10/1733359287165/Put/seqid=0 2024-12-05T00:41:27,323 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d006db90ff714cd08929213b7e92eeb9 is 50, key is test_row_0/A:col10/1733359287165/Put/seqid=0 2024-12-05T00:41:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359347329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359347329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359347330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359347331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741908_1084 (size=13085) 2024-12-05T00:41:27,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359347336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741907_1083 (size=13085) 2024-12-05T00:41:27,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741906_1082 (size=12301) 2024-12-05T00:41:27,367 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/822e39df63b04414b7b463dbd4fa405b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/822e39df63b04414b7b463dbd4fa405b 2024-12-05T00:41:27,374 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 822e39df63b04414b7b463dbd4fa405b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:27,374 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:27,374 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359287305; duration=0sec 2024-12-05T00:41:27,374 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:27,374 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:27,374 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:27,375 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:27,376 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:27,376 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,376 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b2108f2fe9e643beb895636bc9684653, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/560698b06aef4e5184a76f5fcaaac50d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/13ef184fda124cea9c5c29e2f1c6e512] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.7 K 2024-12-05T00:41:27,376 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b2108f2fe9e643beb895636bc9684653, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733359284723 2024-12-05T00:41:27,377 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 560698b06aef4e5184a76f5fcaaac50d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733359285882 2024-12-05T00:41:27,377 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ef184fda124cea9c5c29e2f1c6e512, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733359287162 2024-12-05T00:41:27,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T00:41:27,389 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#72 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:27,390 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/03678d0df214445babc475f393ef1b0c is 50, key is test_row_0/C:col10/1733359287165/Put/seqid=0 2024-12-05T00:41:27,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741909_1085 (size=13085) 2024-12-05T00:41:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359347441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359347442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359347442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359347442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359347452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359347652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359347652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359347652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359347652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359347661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:27,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T00:41:27,748 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/d006db90ff714cd08929213b7e92eeb9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d006db90ff714cd08929213b7e92eeb9 2024-12-05T00:41:27,754 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into d006db90ff714cd08929213b7e92eeb9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:27,754 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:27,754 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359287305; duration=0sec 2024-12-05T00:41:27,754 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:27,754 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:27,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/db0f3254913e4b839496e20aedf34599 2024-12-05T00:41:27,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/fdff4806de804b8f99fc1e52143340f9 is 50, key is test_row_0/B:col10/1733359287239/Put/seqid=0 2024-12-05T00:41:27,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741910_1086 (size=12301) 2024-12-05T00:41:27,803 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/03678d0df214445babc475f393ef1b0c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/03678d0df214445babc475f393ef1b0c 2024-12-05T00:41:27,808 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 03678d0df214445babc475f393ef1b0c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:27,808 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:27,808 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359287306; duration=0sec 2024-12-05T00:41:27,808 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:27,808 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:27,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359347962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359347962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359347962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359347962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359347971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:27,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:27,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:27,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:27,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:28,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:28,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/fdff4806de804b8f99fc1e52143340f9 2024-12-05T00:41:28,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/e8fbcf2003ee4438b4e8b1706d601044 is 50, key is test_row_0/C:col10/1733359287239/Put/seqid=0 2024-12-05T00:41:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741911_1087 (size=12301) 2024-12-05T00:41:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T00:41:28,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359348471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:28,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359348471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:28,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359348471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:28,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359348471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359348482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:28,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/e8fbcf2003ee4438b4e8b1706d601044 2024-12-05T00:41:28,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/db0f3254913e4b839496e20aedf34599 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/db0f3254913e4b839496e20aedf34599 2024-12-05T00:41:28,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:28,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/db0f3254913e4b839496e20aedf34599, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T00:41:28,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/fdff4806de804b8f99fc1e52143340f9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/fdff4806de804b8f99fc1e52143340f9 2024-12-05T00:41:28,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/fdff4806de804b8f99fc1e52143340f9, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T00:41:28,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/e8fbcf2003ee4438b4e8b1706d601044 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e8fbcf2003ee4438b4e8b1706d601044 2024-12-05T00:41:28,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e8fbcf2003ee4438b4e8b1706d601044, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T00:41:28,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.79 KB/136996, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for d5061eee39269f2ef93c654f6e1034a0 in 1314ms, sequenceid=360, compaction requested=false 2024-12-05T00:41:28,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:28,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T00:41:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,757 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:41:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/2680142eb82f4f96bc65eb9e08caaa05 is 50, key is test_row_0/A:col10/1733359287328/Put/seqid=0 2024-12-05T00:41:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741912_1088 (size=12301) 2024-12-05T00:41:28,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/2680142eb82f4f96bc65eb9e08caaa05 2024-12-05T00:41:28,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/b5e596d940f040249422d80b65368264 is 50, key is test_row_0/B:col10/1733359287328/Put/seqid=0 2024-12-05T00:41:28,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741913_1089 (size=12301) 2024-12-05T00:41:28,814 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/b5e596d940f040249422d80b65368264 2024-12-05T00:41:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/b03c5c232d9645e5ad111380805c8944 is 50, key is test_row_0/C:col10/1733359287328/Put/seqid=0 2024-12-05T00:41:28,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741914_1090 (size=12301) 2024-12-05T00:41:28,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/b03c5c232d9645e5ad111380805c8944 2024-12-05T00:41:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/2680142eb82f4f96bc65eb9e08caaa05 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2680142eb82f4f96bc65eb9e08caaa05 2024-12-05T00:41:28,863 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2680142eb82f4f96bc65eb9e08caaa05, entries=150, sequenceid=373, filesize=12.0 K 2024-12-05T00:41:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/b5e596d940f040249422d80b65368264 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b5e596d940f040249422d80b65368264 2024-12-05T00:41:28,874 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b5e596d940f040249422d80b65368264, entries=150, sequenceid=373, filesize=12.0 K 2024-12-05T00:41:28,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/b03c5c232d9645e5ad111380805c8944 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b03c5c232d9645e5ad111380805c8944 2024-12-05T00:41:28,884 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b03c5c232d9645e5ad111380805c8944, entries=150, sequenceid=373, filesize=12.0 K 2024-12-05T00:41:28,887 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for d5061eee39269f2ef93c654f6e1034a0 in 129ms, sequenceid=373, compaction requested=true 2024-12-05T00:41:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T00:41:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-05T00:41:28,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T00:41:28,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-12-05T00:41:28,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.8340 sec 2024-12-05T00:41:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T00:41:29,212 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:29,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-05T00:41:29,215 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T00:41:29,216 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:29,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T00:41:29,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T00:41:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-05T00:41:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-05T00:41:29,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T00:41:29,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-05T00:41:29,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 161 msec 2024-12-05T00:41:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:29,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:41:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:29,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/22b8fe30e74544c6aef9c094c87b3c1b is 50, key is test_row_0/A:col10/1733359289486/Put/seqid=0 2024-12-05T00:41:29,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741915_1091 (size=12301) 2024-12-05T00:41:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T00:41:29,532 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:29,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-05T00:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T00:41:29,535 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:29,535 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:29,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:29,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359289507, max timestamp=1733359289546Num uniques 859; 2024-12-05T00:41:29,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359289507, max timestamp=1733359289546Num uniques 859; 2024-12-05T00:41:29,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:29,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:29,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359289507, max timestamp=1733359289546Num uniques 859; 2024-12-05T00:41:29,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:29,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359349576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359349578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359349580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359349581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359349585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T00:41:29,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T00:41:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359349692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359349692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359349692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359349692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359349701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T00:41:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T00:41:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359349902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359349902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359349902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359349902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/22b8fe30e74544c6aef9c094c87b3c1b 2024-12-05T00:41:29,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9 is 50, key is test_row_0/B:col10/1733359289486/Put/seqid=0 2024-12-05T00:41:29,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359349911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:29,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:29,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:29,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741916_1092 (size=12301) 2024-12-05T00:41:29,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9 2024-12-05T00:41:29,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/e43d0abcafc541acaee84f0f77d2ca0a is 50, key is test_row_0/C:col10/1733359289486/Put/seqid=0 2024-12-05T00:41:29,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741917_1093 (size=12301) 2024-12-05T00:41:29,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T00:41:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:29,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T00:41:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T00:41:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359350212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359350212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359350212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359350212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359350221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T00:41:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:30,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/e43d0abcafc541acaee84f0f77d2ca0a 2024-12-05T00:41:30,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/22b8fe30e74544c6aef9c094c87b3c1b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/22b8fe30e74544c6aef9c094c87b3c1b 2024-12-05T00:41:30,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/22b8fe30e74544c6aef9c094c87b3c1b, entries=150, sequenceid=385, filesize=12.0 K 2024-12-05T00:41:30,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9 2024-12-05T00:41:30,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9, entries=150, sequenceid=385, filesize=12.0 K 2024-12-05T00:41:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/e43d0abcafc541acaee84f0f77d2ca0a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e43d0abcafc541acaee84f0f77d2ca0a 2024-12-05T00:41:30,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e43d0abcafc541acaee84f0f77d2ca0a, entries=150, sequenceid=385, filesize=12.0 K 2024-12-05T00:41:30,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d5061eee39269f2ef93c654f6e1034a0 in 879ms, sequenceid=385, compaction requested=true 2024-12-05T00:41:30,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:30,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:30,373 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:30,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:30,374 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:30,375 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:30,375 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:30,375 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:30,375 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:30,375 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,375 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,375 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/822e39df63b04414b7b463dbd4fa405b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/fdff4806de804b8f99fc1e52143340f9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b5e596d940f040249422d80b65368264, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=48.8 K 2024-12-05T00:41:30,375 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d006db90ff714cd08929213b7e92eeb9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/db0f3254913e4b839496e20aedf34599, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2680142eb82f4f96bc65eb9e08caaa05, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/22b8fe30e74544c6aef9c094c87b3c1b] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=48.8 K 2024-12-05T00:41:30,375 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 822e39df63b04414b7b463dbd4fa405b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733359287162 2024-12-05T00:41:30,375 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d006db90ff714cd08929213b7e92eeb9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733359287162 2024-12-05T00:41:30,376 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting db0f3254913e4b839496e20aedf34599, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733359287239 2024-12-05T00:41:30,376 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting fdff4806de804b8f99fc1e52143340f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733359287239 2024-12-05T00:41:30,376 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2680142eb82f4f96bc65eb9e08caaa05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733359287326 2024-12-05T00:41:30,376 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e596d940f040249422d80b65368264, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733359287326 2024-12-05T00:41:30,376 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b8fe30e74544c6aef9c094c87b3c1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733359289486 2024-12-05T00:41:30,376 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b9cde0f1f104e1dbb62bbcdc6abe5a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733359289486 2024-12-05T00:41:30,397 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:30,398 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:30,398 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/77c128e3a2e14430b57a5b9b7dca4361 is 50, key is test_row_0/A:col10/1733359289486/Put/seqid=0 2024-12-05T00:41:30,398 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/bf5645e6a1c84c6ca6ea9a07d8783cdc is 50, key is test_row_0/B:col10/1733359289486/Put/seqid=0 2024-12-05T00:41:30,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741918_1094 (size=13221) 2024-12-05T00:41:30,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741919_1095 (size=13221) 2024-12-05T00:41:30,422 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/77c128e3a2e14430b57a5b9b7dca4361 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/77c128e3a2e14430b57a5b9b7dca4361 2024-12-05T00:41:30,428 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into 77c128e3a2e14430b57a5b9b7dca4361(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:30,428 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:30,428 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=12, startTime=1733359290373; duration=0sec 2024-12-05T00:41:30,428 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:30,428 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:30,428 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:30,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:30,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:30,430 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,430 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/03678d0df214445babc475f393ef1b0c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e8fbcf2003ee4438b4e8b1706d601044, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b03c5c232d9645e5ad111380805c8944, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e43d0abcafc541acaee84f0f77d2ca0a] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=48.8 K 2024-12-05T00:41:30,431 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03678d0df214445babc475f393ef1b0c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733359287162 2024-12-05T00:41:30,431 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8fbcf2003ee4438b4e8b1706d601044, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733359287239 2024-12-05T00:41:30,431 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03c5c232d9645e5ad111380805c8944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733359287326 2024-12-05T00:41:30,432 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43d0abcafc541acaee84f0f77d2ca0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733359289486 2024-12-05T00:41:30,441 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:30,442 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/728ae3f499174337acde3c2e583b0234 is 50, key is test_row_0/C:col10/1733359289486/Put/seqid=0 2024-12-05T00:41:30,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741920_1096 (size=13221) 2024-12-05T00:41:30,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-05T00:41:30,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T00:41:30,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:30,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:30,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:30,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:30,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:30,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:30,458 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/728ae3f499174337acde3c2e583b0234 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/728ae3f499174337acde3c2e583b0234 2024-12-05T00:41:30,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fcc3864cd779465fa77cd1ef7e3f4ff1 is 50, key is test_row_0/A:col10/1733359289579/Put/seqid=0 2024-12-05T00:41:30,467 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 728ae3f499174337acde3c2e583b0234(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:30,467 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:30,467 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=12, startTime=1733359290374; duration=0sec 2024-12-05T00:41:30,467 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:30,467 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:30,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741921_1097 (size=12301) 2024-12-05T00:41:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T00:41:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:30,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359350730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359350731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359350732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359350732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359350732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,820 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/bf5645e6a1c84c6ca6ea9a07d8783cdc as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/bf5645e6a1c84c6ca6ea9a07d8783cdc 2024-12-05T00:41:30,832 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into bf5645e6a1c84c6ca6ea9a07d8783cdc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:30,833 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:30,833 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=12, startTime=1733359290373; duration=0sec 2024-12-05T00:41:30,833 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:30,833 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:30,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359350842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359350843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359350843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359350844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:30,877 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fcc3864cd779465fa77cd1ef7e3f4ff1 2024-12-05T00:41:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ddeeadc600d74e4f9f354f82a0e20c9f is 50, key is test_row_0/B:col10/1733359289579/Put/seqid=0 2024-12-05T00:41:30,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741922_1098 (size=12301) 2024-12-05T00:41:30,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ddeeadc600d74e4f9f354f82a0e20c9f 2024-12-05T00:41:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/16b2dfd46df34995ae8eabc6b6ab16fc is 50, key is test_row_0/C:col10/1733359289579/Put/seqid=0 2024-12-05T00:41:30,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741923_1099 (size=12301) 2024-12-05T00:41:30,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/16b2dfd46df34995ae8eabc6b6ab16fc 2024-12-05T00:41:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fcc3864cd779465fa77cd1ef7e3f4ff1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fcc3864cd779465fa77cd1ef7e3f4ff1 2024-12-05T00:41:30,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fcc3864cd779465fa77cd1ef7e3f4ff1, entries=150, sequenceid=422, filesize=12.0 K 2024-12-05T00:41:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ddeeadc600d74e4f9f354f82a0e20c9f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddeeadc600d74e4f9f354f82a0e20c9f 2024-12-05T00:41:30,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddeeadc600d74e4f9f354f82a0e20c9f, entries=150, sequenceid=422, filesize=12.0 K 2024-12-05T00:41:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/16b2dfd46df34995ae8eabc6b6ab16fc as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/16b2dfd46df34995ae8eabc6b6ab16fc 2024-12-05T00:41:30,953 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/16b2dfd46df34995ae8eabc6b6ab16fc, entries=150, sequenceid=422, filesize=12.0 K 2024-12-05T00:41:30,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for d5061eee39269f2ef93c654f6e1034a0 in 501ms, sequenceid=422, compaction requested=false 2024-12-05T00:41:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-05T00:41:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-05T00:41:30,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T00:41:30,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-12-05T00:41:30,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4270 sec 2024-12-05T00:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:31,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:41:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:31,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fff51673fe8c49699d92cc1e5e7faab2 is 50, key is test_row_0/A:col10/1733359291052/Put/seqid=0 2024-12-05T00:41:31,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741924_1100 (size=17181) 2024-12-05T00:41:31,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359351101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359351107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359351109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359351109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359351212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359351222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359351222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359351222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359351422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359351432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359351432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359351432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fff51673fe8c49699d92cc1e5e7faab2 2024-12-05T00:41:31,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/b65640b8247b41e7b7c2aa215e805658 is 50, key is test_row_0/B:col10/1733359291052/Put/seqid=0 2024-12-05T00:41:31,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741925_1101 (size=12301) 2024-12-05T00:41:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T00:41:31,681 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:31,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-05T00:41:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T00:41:31,683 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:31,684 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:31,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:31,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359351732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359351742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359351742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359351742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359351751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:31,758 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in d5061eee39269f2ef93c654f6e1034a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T00:41:31,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T00:41:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:31,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:31,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/b65640b8247b41e7b7c2aa215e805658 2024-12-05T00:41:31,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/d4c8788e1f59419cb92ca3b8e6cdd4c6 is 50, key is test_row_0/C:col10/1733359291052/Put/seqid=0 2024-12-05T00:41:31,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741926_1102 (size=12301) 2024-12-05T00:41:31,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T00:41:31,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:31,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:31,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:31,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T00:41:32,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T00:41:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359352241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:32,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359352252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:32,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359352252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359352252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:32,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T00:41:32,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:32,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T00:41:32,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/d4c8788e1f59419cb92ca3b8e6cdd4c6 2024-12-05T00:41:32,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fff51673fe8c49699d92cc1e5e7faab2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fff51673fe8c49699d92cc1e5e7faab2 2024-12-05T00:41:32,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fff51673fe8c49699d92cc1e5e7faab2, entries=250, sequenceid=438, filesize=16.8 K 2024-12-05T00:41:32,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/b65640b8247b41e7b7c2aa215e805658 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b65640b8247b41e7b7c2aa215e805658 2024-12-05T00:41:32,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b65640b8247b41e7b7c2aa215e805658, entries=150, sequenceid=438, filesize=12.0 K 2024-12-05T00:41:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/d4c8788e1f59419cb92ca3b8e6cdd4c6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d4c8788e1f59419cb92ca3b8e6cdd4c6 2024-12-05T00:41:32,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d4c8788e1f59419cb92ca3b8e6cdd4c6, entries=150, sequenceid=438, filesize=12.0 K 2024-12-05T00:41:32,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for d5061eee39269f2ef93c654f6e1034a0 in 1355ms, sequenceid=438, compaction requested=true 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:32,412 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T00:41:32,412 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:32,414 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:32,415 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:32,415 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,416 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/bf5645e6a1c84c6ca6ea9a07d8783cdc, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddeeadc600d74e4f9f354f82a0e20c9f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b65640b8247b41e7b7c2aa215e805658] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.9 K 2024-12-05T00:41:32,416 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:32,416 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:32,416 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,416 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/77c128e3a2e14430b57a5b9b7dca4361, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fcc3864cd779465fa77cd1ef7e3f4ff1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fff51673fe8c49699d92cc1e5e7faab2] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=41.7 K 2024-12-05T00:41:32,417 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5645e6a1c84c6ca6ea9a07d8783cdc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733359289486 2024-12-05T00:41:32,418 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c128e3a2e14430b57a5b9b7dca4361, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733359289486 2024-12-05T00:41:32,419 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc3864cd779465fa77cd1ef7e3f4ff1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733359289577 2024-12-05T00:41:32,419 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ddeeadc600d74e4f9f354f82a0e20c9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733359289577 2024-12-05T00:41:32,420 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff51673fe8c49699d92cc1e5e7faab2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1733359290730 2024-12-05T00:41:32,420 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b65640b8247b41e7b7c2aa215e805658, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1733359290730 2024-12-05T00:41:32,442 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:32,443 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ac483d4ded034843956d4ec98d855efa is 50, key is test_row_0/B:col10/1733359291052/Put/seqid=0 2024-12-05T00:41:32,452 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:32,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-05T00:41:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,452 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:41:32,452 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fd5639bf5e014092ae0256d5848dbb35 is 50, key is test_row_0/A:col10/1733359291052/Put/seqid=0 2024-12-05T00:41:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:32,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/50e960d13c44412096d0472d1a62e7cb is 50, key is test_row_0/A:col10/1733359291106/Put/seqid=0 2024-12-05T00:41:32,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741927_1103 (size=13323) 2024-12-05T00:41:32,493 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/ac483d4ded034843956d4ec98d855efa as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ac483d4ded034843956d4ec98d855efa 2024-12-05T00:41:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741928_1104 (size=13323) 2024-12-05T00:41:32,513 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into ac483d4ded034843956d4ec98d855efa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:32,513 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:32,513 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359292412; duration=0sec 2024-12-05T00:41:32,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:32,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:32,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:32,516 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:32,516 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:32,516 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:32,517 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/728ae3f499174337acde3c2e583b0234, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/16b2dfd46df34995ae8eabc6b6ab16fc, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d4c8788e1f59419cb92ca3b8e6cdd4c6] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=36.9 K 2024-12-05T00:41:32,520 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 728ae3f499174337acde3c2e583b0234, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733359289486 2024-12-05T00:41:32,522 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b2dfd46df34995ae8eabc6b6ab16fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733359289577 2024-12-05T00:41:32,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c8788e1f59419cb92ca3b8e6cdd4c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1733359290730 2024-12-05T00:41:32,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741929_1105 (size=12301) 2024-12-05T00:41:32,538 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/50e960d13c44412096d0472d1a62e7cb 2024-12-05T00:41:32,546 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:32,547 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/d536af276a6549768e33cdcdc966ded0 is 50, key is test_row_0/C:col10/1733359291052/Put/seqid=0 2024-12-05T00:41:32,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/1be31ad9b8f04436936c23c7caa92beb is 50, key is test_row_0/B:col10/1733359291106/Put/seqid=0 2024-12-05T00:41:32,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741930_1106 (size=13323) 2024-12-05T00:41:32,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741931_1107 (size=12301) 2024-12-05T00:41:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/1be31ad9b8f04436936c23c7caa92beb 2024-12-05T00:41:32,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/9aa8930fd4b74b6e8f81552e49d3fd13 is 50, key is test_row_0/C:col10/1733359291106/Put/seqid=0 2024-12-05T00:41:32,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741932_1108 (size=12301) 2024-12-05T00:41:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T00:41:32,915 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/fd5639bf5e014092ae0256d5848dbb35 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fd5639bf5e014092ae0256d5848dbb35 2024-12-05T00:41:32,921 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into fd5639bf5e014092ae0256d5848dbb35(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:32,921 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:32,921 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359292412; duration=0sec 2024-12-05T00:41:32,922 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:32,922 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:32,971 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/d536af276a6549768e33cdcdc966ded0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d536af276a6549768e33cdcdc966ded0 2024-12-05T00:41:32,977 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into d536af276a6549768e33cdcdc966ded0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:32,977 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:32,977 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359292412; duration=0sec 2024-12-05T00:41:32,977 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:32,977 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:33,008 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/9aa8930fd4b74b6e8f81552e49d3fd13 2024-12-05T00:41:33,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/50e960d13c44412096d0472d1a62e7cb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/50e960d13c44412096d0472d1a62e7cb 2024-12-05T00:41:33,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/50e960d13c44412096d0472d1a62e7cb, entries=150, sequenceid=457, filesize=12.0 K 2024-12-05T00:41:33,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/1be31ad9b8f04436936c23c7caa92beb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/1be31ad9b8f04436936c23c7caa92beb 2024-12-05T00:41:33,026 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/1be31ad9b8f04436936c23c7caa92beb, entries=150, sequenceid=457, filesize=12.0 K 2024-12-05T00:41:33,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/9aa8930fd4b74b6e8f81552e49d3fd13 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9aa8930fd4b74b6e8f81552e49d3fd13 2024-12-05T00:41:33,033 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9aa8930fd4b74b6e8f81552e49d3fd13, entries=150, sequenceid=457, filesize=12.0 K 2024-12-05T00:41:33,034 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for d5061eee39269f2ef93c654f6e1034a0 in 582ms, sequenceid=457, compaction requested=false 2024-12-05T00:41:33,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:33,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:33,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-05T00:41:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-05T00:41:33,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T00:41:33,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3510 sec 2024-12-05T00:41:33,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.3560 sec 2024-12-05T00:41:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:33,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:41:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:33,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/ff64a468c9574701bbb349ef3a10d93c is 50, key is test_row_0/A:col10/1733359293267/Put/seqid=0 2024-12-05T00:41:33,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741933_1109 (size=14741) 2024-12-05T00:41:33,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/ff64a468c9574701bbb349ef3a10d93c 2024-12-05T00:41:33,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359353292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359353292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359353294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a7a398798fcb4ea38151c3565bbb4a7f is 50, key is test_row_0/B:col10/1733359293267/Put/seqid=0 2024-12-05T00:41:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359353295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741934_1110 (size=12301) 2024-12-05T00:41:33,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359353402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359353411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359353411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359353411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359353612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359353621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359353621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359353621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a7a398798fcb4ea38151c3565bbb4a7f 2024-12-05T00:41:33,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/93c30de9024342649cc4f33d27809434 is 50, key is test_row_0/C:col10/1733359293267/Put/seqid=0 2024-12-05T00:41:33,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741935_1111 (size=12301) 2024-12-05T00:41:33,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58624 deadline: 1733359353782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,783 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in d5061eee39269f2ef93c654f6e1034a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T00:41:33,832 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:33,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-05T00:41:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T00:41:33,834 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:33,835 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:33,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:33,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58674 deadline: 1733359353921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58612 deadline: 1733359353931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58640 deadline: 1733359353931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58600 deadline: 1733359353931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:41:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T00:41:33,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T00:41:33,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:33,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:33,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:33,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:33,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:34,119 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-05T00:41:34,119 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,121 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-05T00:41:34,121 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,121 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-05T00:41:34,121 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,123 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-05T00:41:34,123 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,124 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,124 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,124 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,124 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,125 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,125 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,125 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,125 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/93c30de9024342649cc4f33d27809434 2024-12-05T00:41:34,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/ff64a468c9574701bbb349ef3a10d93c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ff64a468c9574701bbb349ef3a10d93c 2024-12-05T00:41:34,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T00:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:34,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:34,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ff64a468c9574701bbb349ef3a10d93c, entries=200, sequenceid=472, filesize=14.4 K 2024-12-05T00:41:34,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/a7a398798fcb4ea38151c3565bbb4a7f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a7a398798fcb4ea38151c3565bbb4a7f 2024-12-05T00:41:34,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a7a398798fcb4ea38151c3565bbb4a7f, entries=150, sequenceid=472, filesize=12.0 K 2024-12-05T00:41:34,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/93c30de9024342649cc4f33d27809434 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/93c30de9024342649cc4f33d27809434 2024-12-05T00:41:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T00:41:34,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/93c30de9024342649cc4f33d27809434, entries=150, sequenceid=472, filesize=12.0 K 2024-12-05T00:41:34,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d5061eee39269f2ef93c654f6e1034a0 in 887ms, sequenceid=472, compaction requested=true 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:34,156 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:34,156 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5061eee39269f2ef93c654f6e1034a0:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:34,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:34,157 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:34,157 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:34,157 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/A is initiating minor compaction (all files) 2024-12-05T00:41:34,157 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/B is initiating minor compaction (all files) 2024-12-05T00:41:34,157 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/A in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:34,157 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/B in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:34,157 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ac483d4ded034843956d4ec98d855efa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/1be31ad9b8f04436936c23c7caa92beb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a7a398798fcb4ea38151c3565bbb4a7f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=37.0 K 2024-12-05T00:41:34,157 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fd5639bf5e014092ae0256d5848dbb35, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/50e960d13c44412096d0472d1a62e7cb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ff64a468c9574701bbb349ef3a10d93c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=39.4 K 2024-12-05T00:41:34,157 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ac483d4ded034843956d4ec98d855efa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1733359290730 2024-12-05T00:41:34,157 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5639bf5e014092ae0256d5848dbb35, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1733359290730 2024-12-05T00:41:34,158 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be31ad9b8f04436936c23c7caa92beb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733359291073 2024-12-05T00:41:34,158 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e960d13c44412096d0472d1a62e7cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733359291073 2024-12-05T00:41:34,158 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a7a398798fcb4ea38151c3565bbb4a7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733359293267 2024-12-05T00:41:34,158 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff64a468c9574701bbb349ef3a10d93c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733359293264 2024-12-05T00:41:34,164 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#A#compaction#99 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:34,164 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#B#compaction#100 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:34,165 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/cd3041b2a1fb4318a1277fe27f2469bf is 50, key is test_row_0/A:col10/1733359293267/Put/seqid=0 2024-12-05T00:41:34,165 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/46edcd81951b49299b93daa325a88354 is 50, key is test_row_0/B:col10/1733359293267/Put/seqid=0 2024-12-05T00:41:34,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741936_1112 (size=13425) 2024-12-05T00:41:34,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741937_1113 (size=13425) 2024-12-05T00:41:34,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-05T00:41:34,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:34,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:41:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:34,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/e82eaf179386488b949bf94d6c4ccb29 is 50, key is test_row_0/A:col10/1733359293288/Put/seqid=0 2024-12-05T00:41:34,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741938_1114 (size=12301) 2024-12-05T00:41:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:34,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. as already flushing 2024-12-05T00:41:34,437 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-05T00:41:34,437 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,437 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,438 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,445 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-05T00:41:34,445 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-05T00:41:34,445 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-05T00:41:34,445 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,445 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,445 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:34,445 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,445 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,445 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,445 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,445 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,445 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:34,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T00:41:34,582 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/46edcd81951b49299b93daa325a88354 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/46edcd81951b49299b93daa325a88354 2024-12-05T00:41:34,582 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/cd3041b2a1fb4318a1277fe27f2469bf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/cd3041b2a1fb4318a1277fe27f2469bf 2024-12-05T00:41:34,589 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/A of d5061eee39269f2ef93c654f6e1034a0 into cd3041b2a1fb4318a1277fe27f2469bf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:34,589 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/B of d5061eee39269f2ef93c654f6e1034a0 into 46edcd81951b49299b93daa325a88354(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:34,589 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:34,589 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:34,589 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/A, priority=13, startTime=1733359294156; duration=0sec 2024-12-05T00:41:34,589 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/B, priority=13, startTime=1733359294156; duration=0sec 2024-12-05T00:41:34,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:34,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:B 2024-12-05T00:41:34,590 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:34,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:34,590 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:A 2024-12-05T00:41:34,591 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:34,591 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): d5061eee39269f2ef93c654f6e1034a0/C is initiating minor compaction (all files) 2024-12-05T00:41:34,591 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5061eee39269f2ef93c654f6e1034a0/C in TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:34,591 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d536af276a6549768e33cdcdc966ded0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9aa8930fd4b74b6e8f81552e49d3fd13, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/93c30de9024342649cc4f33d27809434] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp, totalSize=37.0 K 2024-12-05T00:41:34,591 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d536af276a6549768e33cdcdc966ded0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1733359290730 2024-12-05T00:41:34,592 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa8930fd4b74b6e8f81552e49d3fd13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733359291073 2024-12-05T00:41:34,592 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c30de9024342649cc4f33d27809434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733359293267 2024-12-05T00:41:34,598 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5061eee39269f2ef93c654f6e1034a0#C#compaction#102 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:34,599 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/928c4334b17f4b4090a87e5b34f0a289 is 50, key is test_row_0/C:col10/1733359293267/Put/seqid=0 2024-12-05T00:41:34,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741939_1115 (size=13425) 2024-12-05T00:41:34,713 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/e82eaf179386488b949bf94d6c4ccb29 2024-12-05T00:41:34,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/77acd4dd654b4380b35603d3c434051c is 50, key is test_row_0/B:col10/1733359293288/Put/seqid=0 2024-12-05T00:41:34,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741940_1116 (size=12301) 2024-12-05T00:41:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T00:41:35,017 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/928c4334b17f4b4090a87e5b34f0a289 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/928c4334b17f4b4090a87e5b34f0a289 2024-12-05T00:41:35,024 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5061eee39269f2ef93c654f6e1034a0/C of d5061eee39269f2ef93c654f6e1034a0 into 928c4334b17f4b4090a87e5b34f0a289(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:35,024 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:35,024 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., storeName=d5061eee39269f2ef93c654f6e1034a0/C, priority=13, startTime=1733359294156; duration=0sec 2024-12-05T00:41:35,024 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:35,024 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5061eee39269f2ef93c654f6e1034a0:C 2024-12-05T00:41:35,134 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/77acd4dd654b4380b35603d3c434051c 2024-12-05T00:41:35,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/48740871feb94436817d1aae323dbd6a is 50, key is test_row_0/C:col10/1733359293288/Put/seqid=0 2024-12-05T00:41:35,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741941_1117 (size=12301) 2024-12-05T00:41:35,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/48740871feb94436817d1aae323dbd6a 2024-12-05T00:41:35,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/e82eaf179386488b949bf94d6c4ccb29 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/e82eaf179386488b949bf94d6c4ccb29 2024-12-05T00:41:35,562 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/e82eaf179386488b949bf94d6c4ccb29, entries=150, sequenceid=492, filesize=12.0 K 2024-12-05T00:41:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/77acd4dd654b4380b35603d3c434051c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77acd4dd654b4380b35603d3c434051c 2024-12-05T00:41:35,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77acd4dd654b4380b35603d3c434051c, entries=150, sequenceid=492, filesize=12.0 K 2024-12-05T00:41:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/48740871feb94436817d1aae323dbd6a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/48740871feb94436817d1aae323dbd6a 2024-12-05T00:41:35,573 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/48740871feb94436817d1aae323dbd6a, entries=150, sequenceid=492, filesize=12.0 K 2024-12-05T00:41:35,574 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for d5061eee39269f2ef93c654f6e1034a0 in 1279ms, sequenceid=492, compaction requested=false 2024-12-05T00:41:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for d5061eee39269f2ef93c654f6e1034a0: 2024-12-05T00:41:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-05T00:41:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-05T00:41:35,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-05T00:41:35,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-12-05T00:41:35,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.7460 sec 2024-12-05T00:41:35,590 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T00:41:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T00:41:35,993 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:37,793 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-05T00:41:37,793 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:41:37,793 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:37,793 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:37,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:37,793 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3996 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4046 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1808 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5418 rows 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1807 2024-12-05T00:41:37,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5421 rows 2024-12-05T00:41:37,794 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:41:37,795 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:41:37,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:37,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:37,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:41:37,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T00:41:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:37,806 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359297806"}]},"ts":"1733359297806"} 2024-12-05T00:41:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T00:41:37,808 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T00:41:37,809 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T00:41:37,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:41:37,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, UNASSIGN}] 2024-12-05T00:41:37,816 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, UNASSIGN 2024-12-05T00:41:37,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=d5061eee39269f2ef93c654f6e1034a0, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:37,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, UNASSIGN because future has completed 2024-12-05T00:41:37,820 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:41:37,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:41:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T00:41:37,976 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:37,976 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:41:37,977 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing d5061eee39269f2ef93c654f6e1034a0, disabling compactions & flushes 2024-12-05T00:41:37,977 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:37,977 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:37,977 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. after waiting 0 ms 2024-12-05T00:41:37,977 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:37,977 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing d5061eee39269f2ef93c654f6e1034a0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T00:41:37,977 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=A 2024-12-05T00:41:37,978 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:37,978 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=B 2024-12-05T00:41:37,978 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:37,978 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5061eee39269f2ef93c654f6e1034a0, store=C 2024-12-05T00:41:37,978 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:37,982 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/8bda1d894a494663aeaae61ff0b89ea0 is 50, key is test_row_0/A:col10/1733359294433/Put/seqid=0 2024-12-05T00:41:37,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741942_1118 (size=12301) 2024-12-05T00:41:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T00:41:38,388 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/8bda1d894a494663aeaae61ff0b89ea0 2024-12-05T00:41:38,405 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/05109376cfa7418d8eb99ea16b8f696d is 50, key is test_row_0/B:col10/1733359294433/Put/seqid=0 2024-12-05T00:41:38,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741943_1119 (size=12301) 2024-12-05T00:41:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T00:41:38,810 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/05109376cfa7418d8eb99ea16b8f696d 2024-12-05T00:41:38,825 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/062177a543544cf39bb90288e78b8cf8 is 50, key is test_row_0/C:col10/1733359294433/Put/seqid=0 2024-12-05T00:41:38,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741944_1120 (size=12301) 2024-12-05T00:41:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T00:41:39,234 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/062177a543544cf39bb90288e78b8cf8 2024-12-05T00:41:39,244 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/A/8bda1d894a494663aeaae61ff0b89ea0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/8bda1d894a494663aeaae61ff0b89ea0 2024-12-05T00:41:39,251 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/8bda1d894a494663aeaae61ff0b89ea0, entries=150, sequenceid=503, filesize=12.0 K 2024-12-05T00:41:39,253 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/B/05109376cfa7418d8eb99ea16b8f696d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/05109376cfa7418d8eb99ea16b8f696d 2024-12-05T00:41:39,260 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/05109376cfa7418d8eb99ea16b8f696d, entries=150, sequenceid=503, filesize=12.0 K 2024-12-05T00:41:39,261 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/.tmp/C/062177a543544cf39bb90288e78b8cf8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/062177a543544cf39bb90288e78b8cf8 2024-12-05T00:41:39,266 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/062177a543544cf39bb90288e78b8cf8, entries=150, sequenceid=503, filesize=12.0 K 2024-12-05T00:41:39,267 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d5061eee39269f2ef93c654f6e1034a0 in 1289ms, sequenceid=503, compaction requested=true 2024-12-05T00:41:39,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/f123214359b34570934d2a5b831bb9ef, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7b1ec73af6c1441fb4c8c0c11047f5e9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/686796c5de87421688d6d223f625a703, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ba56a08413634992b3b54e1821301304, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/68b1e78106c041a8978d1703ecf0a728, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2bb9732de585467f8688e79bc462c5c6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/3934453dc7264668aeba528b6bbfa829, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a005f11ea15d41c8ac7651331135bb5b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/04fa85f7fa3e4ad38371aa87283541eb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/b3712c52f315416992d2ff7b36a64c31, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7827814211a34e95a79104f899c9cd85, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a38de09ddd1042bd970bfebdc8ec2354, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/bb70f763c1aa407c8743c9936e072a57, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7145eac3d0134432b6b32298ba1bc456, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d5101a97b68440d9ac01a827da2484a9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/0c300fd651a047bdb23358991bfc7dc5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/954efd8e901d4baea4dbee63d749d269, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d206981b59e7487da24c7afa4df6e034, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1a43a63369d44e339ec2050fe9c4afb8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7263be73e2fd47bfb4593eff02ed402f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/c94af1b5ce2845ffaeebd0500a2abe8c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/660556d2cc724396b0601d52d989ff51, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d006db90ff714cd08929213b7e92eeb9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1b847741fa894a0b9a356c25c3607a01, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/db0f3254913e4b839496e20aedf34599, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2680142eb82f4f96bc65eb9e08caaa05, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/77c128e3a2e14430b57a5b9b7dca4361, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/22b8fe30e74544c6aef9c094c87b3c1b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fcc3864cd779465fa77cd1ef7e3f4ff1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fff51673fe8c49699d92cc1e5e7faab2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fd5639bf5e014092ae0256d5848dbb35, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/50e960d13c44412096d0472d1a62e7cb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ff64a468c9574701bbb349ef3a10d93c] to archive 2024-12-05T00:41:39,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:41:39,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/f123214359b34570934d2a5b831bb9ef to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/f123214359b34570934d2a5b831bb9ef 2024-12-05T00:41:39,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7b1ec73af6c1441fb4c8c0c11047f5e9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7b1ec73af6c1441fb4c8c0c11047f5e9 2024-12-05T00:41:39,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/686796c5de87421688d6d223f625a703 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/686796c5de87421688d6d223f625a703 2024-12-05T00:41:39,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ba56a08413634992b3b54e1821301304 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ba56a08413634992b3b54e1821301304 2024-12-05T00:41:39,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/68b1e78106c041a8978d1703ecf0a728 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/68b1e78106c041a8978d1703ecf0a728 2024-12-05T00:41:39,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2bb9732de585467f8688e79bc462c5c6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2bb9732de585467f8688e79bc462c5c6 2024-12-05T00:41:39,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/3934453dc7264668aeba528b6bbfa829 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/3934453dc7264668aeba528b6bbfa829 2024-12-05T00:41:39,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a005f11ea15d41c8ac7651331135bb5b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a005f11ea15d41c8ac7651331135bb5b 2024-12-05T00:41:39,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/04fa85f7fa3e4ad38371aa87283541eb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/04fa85f7fa3e4ad38371aa87283541eb 2024-12-05T00:41:39,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/b3712c52f315416992d2ff7b36a64c31 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/b3712c52f315416992d2ff7b36a64c31 2024-12-05T00:41:39,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7827814211a34e95a79104f899c9cd85 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7827814211a34e95a79104f899c9cd85 2024-12-05T00:41:39,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a38de09ddd1042bd970bfebdc8ec2354 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/a38de09ddd1042bd970bfebdc8ec2354 2024-12-05T00:41:39,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/bb70f763c1aa407c8743c9936e072a57 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/bb70f763c1aa407c8743c9936e072a57 2024-12-05T00:41:39,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7145eac3d0134432b6b32298ba1bc456 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7145eac3d0134432b6b32298ba1bc456 2024-12-05T00:41:39,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d5101a97b68440d9ac01a827da2484a9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d5101a97b68440d9ac01a827da2484a9 2024-12-05T00:41:39,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/0c300fd651a047bdb23358991bfc7dc5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/0c300fd651a047bdb23358991bfc7dc5 2024-12-05T00:41:39,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/954efd8e901d4baea4dbee63d749d269 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/954efd8e901d4baea4dbee63d749d269 2024-12-05T00:41:39,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d206981b59e7487da24c7afa4df6e034 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d206981b59e7487da24c7afa4df6e034 2024-12-05T00:41:39,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1a43a63369d44e339ec2050fe9c4afb8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1a43a63369d44e339ec2050fe9c4afb8 2024-12-05T00:41:39,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7263be73e2fd47bfb4593eff02ed402f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/7263be73e2fd47bfb4593eff02ed402f 2024-12-05T00:41:39,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/c94af1b5ce2845ffaeebd0500a2abe8c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/c94af1b5ce2845ffaeebd0500a2abe8c 2024-12-05T00:41:39,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/660556d2cc724396b0601d52d989ff51 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/660556d2cc724396b0601d52d989ff51 2024-12-05T00:41:39,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d006db90ff714cd08929213b7e92eeb9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/d006db90ff714cd08929213b7e92eeb9 2024-12-05T00:41:39,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1b847741fa894a0b9a356c25c3607a01 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/1b847741fa894a0b9a356c25c3607a01 2024-12-05T00:41:39,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/db0f3254913e4b839496e20aedf34599 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/db0f3254913e4b839496e20aedf34599 2024-12-05T00:41:39,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2680142eb82f4f96bc65eb9e08caaa05 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/2680142eb82f4f96bc65eb9e08caaa05 2024-12-05T00:41:39,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/77c128e3a2e14430b57a5b9b7dca4361 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/77c128e3a2e14430b57a5b9b7dca4361 2024-12-05T00:41:39,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/22b8fe30e74544c6aef9c094c87b3c1b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/22b8fe30e74544c6aef9c094c87b3c1b 2024-12-05T00:41:39,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fcc3864cd779465fa77cd1ef7e3f4ff1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fcc3864cd779465fa77cd1ef7e3f4ff1 2024-12-05T00:41:39,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fff51673fe8c49699d92cc1e5e7faab2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fff51673fe8c49699d92cc1e5e7faab2 2024-12-05T00:41:39,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fd5639bf5e014092ae0256d5848dbb35 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/fd5639bf5e014092ae0256d5848dbb35 2024-12-05T00:41:39,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/50e960d13c44412096d0472d1a62e7cb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/50e960d13c44412096d0472d1a62e7cb 2024-12-05T00:41:39,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ff64a468c9574701bbb349ef3a10d93c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/ff64a468c9574701bbb349ef3a10d93c 2024-12-05T00:41:39,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/039d6324df824020bdff715723f051d7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/d397f22b376741bbb561b3818a6af104, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/141f605af57947fb8897fecdd4e42382, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/09f9e6b5218346aeab2be7c3c61ce010, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/03ba6ae7a0844dada3764c923804a44c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/396fd56994464216a6c9d589425cc942, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a262d72339474f4fbe1b786c439abfc2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a6f2c02ceb3a49fdb5c3ad56422fdea6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/eb7680dbf3384ae2ac7ce98e10aa2539, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/933808e545ea46aab2c3950989514639, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddc11aca753c4f468779422941a8190d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/50efc39f4f904adc9a70428e9a0607e0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5304a386f6284f908183a5c24b004b1d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/2ecffd11cbfe442aafb1227d284cab70, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/612bd9a695f34df593b05a7556d2e5e0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/21a82fb576a14e91a17f2a6e0eb29bfb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5cf05883b7d24982baddc8f16b38fa5c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/375148a0a82d4516a616ae45074a7542, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/6829e1c00da346fc89cd76f3cb30d212, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/664c230becca4d1586ca3a90ff79f4bd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77487a49d9074bf383dd83a270179d6d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a2c788bcd52e43739b14eb95228f92a3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/822e39df63b04414b7b463dbd4fa405b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/53b6dd59f77f45a7ac681f5ba6c673f1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/fdff4806de804b8f99fc1e52143340f9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b5e596d940f040249422d80b65368264, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/bf5645e6a1c84c6ca6ea9a07d8783cdc, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddeeadc600d74e4f9f354f82a0e20c9f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ac483d4ded034843956d4ec98d855efa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b65640b8247b41e7b7c2aa215e805658, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/1be31ad9b8f04436936c23c7caa92beb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a7a398798fcb4ea38151c3565bbb4a7f] to archive 2024-12-05T00:41:39,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:41:39,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/039d6324df824020bdff715723f051d7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/039d6324df824020bdff715723f051d7 2024-12-05T00:41:39,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/d397f22b376741bbb561b3818a6af104 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/d397f22b376741bbb561b3818a6af104 2024-12-05T00:41:39,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/141f605af57947fb8897fecdd4e42382 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/141f605af57947fb8897fecdd4e42382 2024-12-05T00:41:39,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/09f9e6b5218346aeab2be7c3c61ce010 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/09f9e6b5218346aeab2be7c3c61ce010 2024-12-05T00:41:39,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/03ba6ae7a0844dada3764c923804a44c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/03ba6ae7a0844dada3764c923804a44c 2024-12-05T00:41:39,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/396fd56994464216a6c9d589425cc942 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/396fd56994464216a6c9d589425cc942 2024-12-05T00:41:39,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a262d72339474f4fbe1b786c439abfc2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a262d72339474f4fbe1b786c439abfc2 2024-12-05T00:41:39,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a6f2c02ceb3a49fdb5c3ad56422fdea6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a6f2c02ceb3a49fdb5c3ad56422fdea6 2024-12-05T00:41:39,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/eb7680dbf3384ae2ac7ce98e10aa2539 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/eb7680dbf3384ae2ac7ce98e10aa2539 2024-12-05T00:41:39,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/933808e545ea46aab2c3950989514639 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/933808e545ea46aab2c3950989514639 2024-12-05T00:41:39,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddc11aca753c4f468779422941a8190d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddc11aca753c4f468779422941a8190d 2024-12-05T00:41:39,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/50efc39f4f904adc9a70428e9a0607e0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/50efc39f4f904adc9a70428e9a0607e0 2024-12-05T00:41:39,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5304a386f6284f908183a5c24b004b1d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5304a386f6284f908183a5c24b004b1d 2024-12-05T00:41:39,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/2ecffd11cbfe442aafb1227d284cab70 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/2ecffd11cbfe442aafb1227d284cab70 2024-12-05T00:41:39,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/612bd9a695f34df593b05a7556d2e5e0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/612bd9a695f34df593b05a7556d2e5e0 2024-12-05T00:41:39,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/21a82fb576a14e91a17f2a6e0eb29bfb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/21a82fb576a14e91a17f2a6e0eb29bfb 2024-12-05T00:41:39,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5cf05883b7d24982baddc8f16b38fa5c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/5cf05883b7d24982baddc8f16b38fa5c 2024-12-05T00:41:39,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/375148a0a82d4516a616ae45074a7542 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/375148a0a82d4516a616ae45074a7542 2024-12-05T00:41:39,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/6829e1c00da346fc89cd76f3cb30d212 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/6829e1c00da346fc89cd76f3cb30d212 2024-12-05T00:41:39,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/664c230becca4d1586ca3a90ff79f4bd to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/664c230becca4d1586ca3a90ff79f4bd 2024-12-05T00:41:39,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77487a49d9074bf383dd83a270179d6d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77487a49d9074bf383dd83a270179d6d 2024-12-05T00:41:39,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a2c788bcd52e43739b14eb95228f92a3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a2c788bcd52e43739b14eb95228f92a3 2024-12-05T00:41:39,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/822e39df63b04414b7b463dbd4fa405b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/822e39df63b04414b7b463dbd4fa405b 2024-12-05T00:41:39,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/53b6dd59f77f45a7ac681f5ba6c673f1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/53b6dd59f77f45a7ac681f5ba6c673f1 2024-12-05T00:41:39,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/fdff4806de804b8f99fc1e52143340f9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/fdff4806de804b8f99fc1e52143340f9 2024-12-05T00:41:39,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b5e596d940f040249422d80b65368264 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b5e596d940f040249422d80b65368264 2024-12-05T00:41:39,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/bf5645e6a1c84c6ca6ea9a07d8783cdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/bf5645e6a1c84c6ca6ea9a07d8783cdc 2024-12-05T00:41:39,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/7b9cde0f1f104e1dbb62bbcdc6abe5a9 2024-12-05T00:41:39,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddeeadc600d74e4f9f354f82a0e20c9f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ddeeadc600d74e4f9f354f82a0e20c9f 2024-12-05T00:41:39,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ac483d4ded034843956d4ec98d855efa to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/ac483d4ded034843956d4ec98d855efa 2024-12-05T00:41:39,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b65640b8247b41e7b7c2aa215e805658 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/b65640b8247b41e7b7c2aa215e805658 2024-12-05T00:41:39,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/1be31ad9b8f04436936c23c7caa92beb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/1be31ad9b8f04436936c23c7caa92beb 2024-12-05T00:41:39,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a7a398798fcb4ea38151c3565bbb4a7f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/a7a398798fcb4ea38151c3565bbb4a7f 2024-12-05T00:41:39,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/59d30bd0acf3447097de50e0cb4c509f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/64feba6352fb4478b726fd1017417702, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/4a130724a39146268f4c186c8f6e7c48, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/25415846c2cc429c89abbdf60e263940, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/aa8fc32a41b84795976f8b1cf7701d83, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/46302e889d064f359f9ae17dd3de895c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8e8b06cb0fa14c56b52b3d21d68cb36e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/a9732f4f058445c39d5d9b75a2fcc909, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/70b45467f2144fe1bd15b750778af596, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/675ff977852544ac8f5f261b906257a5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/026fc4592bc04e18b983b96c8ff7a115, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/165201ada7474c1eb369a84bb7c5f757, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8f8f31da7fa3475d8a413ae428d34faf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/496632f2dcf341b1854c6acadf7861a1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5fd2711b9b64b1485a1ecdcd7ef7f61, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/6b4c32dc2170498c810f5fbd2c223b8e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5bc6ab142e74d1583973a1927c73b79, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/ba747b00f0f043bb8a2768528beeb89a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9ba571c9349b453c9631c71b74de1aef, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b2108f2fe9e643beb895636bc9684653, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/332a5157cc0845be82c4f7a1581b8ec4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/560698b06aef4e5184a76f5fcaaac50d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/03678d0df214445babc475f393ef1b0c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/13ef184fda124cea9c5c29e2f1c6e512, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e8fbcf2003ee4438b4e8b1706d601044, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b03c5c232d9645e5ad111380805c8944, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/728ae3f499174337acde3c2e583b0234, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e43d0abcafc541acaee84f0f77d2ca0a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/16b2dfd46df34995ae8eabc6b6ab16fc, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d536af276a6549768e33cdcdc966ded0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d4c8788e1f59419cb92ca3b8e6cdd4c6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9aa8930fd4b74b6e8f81552e49d3fd13, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/93c30de9024342649cc4f33d27809434] to archive 2024-12-05T00:41:39,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:41:39,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/59d30bd0acf3447097de50e0cb4c509f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/59d30bd0acf3447097de50e0cb4c509f 2024-12-05T00:41:39,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/64feba6352fb4478b726fd1017417702 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/64feba6352fb4478b726fd1017417702 2024-12-05T00:41:39,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/4a130724a39146268f4c186c8f6e7c48 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/4a130724a39146268f4c186c8f6e7c48 2024-12-05T00:41:39,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/25415846c2cc429c89abbdf60e263940 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/25415846c2cc429c89abbdf60e263940 2024-12-05T00:41:39,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/aa8fc32a41b84795976f8b1cf7701d83 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/aa8fc32a41b84795976f8b1cf7701d83 2024-12-05T00:41:39,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/46302e889d064f359f9ae17dd3de895c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/46302e889d064f359f9ae17dd3de895c 2024-12-05T00:41:39,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8e8b06cb0fa14c56b52b3d21d68cb36e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8e8b06cb0fa14c56b52b3d21d68cb36e 2024-12-05T00:41:39,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/a9732f4f058445c39d5d9b75a2fcc909 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/a9732f4f058445c39d5d9b75a2fcc909 2024-12-05T00:41:39,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/70b45467f2144fe1bd15b750778af596 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/70b45467f2144fe1bd15b750778af596 2024-12-05T00:41:39,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/675ff977852544ac8f5f261b906257a5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/675ff977852544ac8f5f261b906257a5 2024-12-05T00:41:39,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/026fc4592bc04e18b983b96c8ff7a115 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/026fc4592bc04e18b983b96c8ff7a115 2024-12-05T00:41:39,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/165201ada7474c1eb369a84bb7c5f757 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/165201ada7474c1eb369a84bb7c5f757 2024-12-05T00:41:39,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8f8f31da7fa3475d8a413ae428d34faf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/8f8f31da7fa3475d8a413ae428d34faf 2024-12-05T00:41:39,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/496632f2dcf341b1854c6acadf7861a1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/496632f2dcf341b1854c6acadf7861a1 2024-12-05T00:41:39,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5fd2711b9b64b1485a1ecdcd7ef7f61 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5fd2711b9b64b1485a1ecdcd7ef7f61 2024-12-05T00:41:39,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/6b4c32dc2170498c810f5fbd2c223b8e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/6b4c32dc2170498c810f5fbd2c223b8e 2024-12-05T00:41:39,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5bc6ab142e74d1583973a1927c73b79 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/c5bc6ab142e74d1583973a1927c73b79 2024-12-05T00:41:39,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/ba747b00f0f043bb8a2768528beeb89a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/ba747b00f0f043bb8a2768528beeb89a 2024-12-05T00:41:39,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9ba571c9349b453c9631c71b74de1aef to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9ba571c9349b453c9631c71b74de1aef 2024-12-05T00:41:39,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b2108f2fe9e643beb895636bc9684653 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b2108f2fe9e643beb895636bc9684653 2024-12-05T00:41:39,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/332a5157cc0845be82c4f7a1581b8ec4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/332a5157cc0845be82c4f7a1581b8ec4 2024-12-05T00:41:39,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/560698b06aef4e5184a76f5fcaaac50d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/560698b06aef4e5184a76f5fcaaac50d 2024-12-05T00:41:39,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/03678d0df214445babc475f393ef1b0c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/03678d0df214445babc475f393ef1b0c 2024-12-05T00:41:39,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/13ef184fda124cea9c5c29e2f1c6e512 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/13ef184fda124cea9c5c29e2f1c6e512 2024-12-05T00:41:39,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e8fbcf2003ee4438b4e8b1706d601044 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e8fbcf2003ee4438b4e8b1706d601044 2024-12-05T00:41:39,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b03c5c232d9645e5ad111380805c8944 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/b03c5c232d9645e5ad111380805c8944 2024-12-05T00:41:39,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/728ae3f499174337acde3c2e583b0234 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/728ae3f499174337acde3c2e583b0234 2024-12-05T00:41:39,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e43d0abcafc541acaee84f0f77d2ca0a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/e43d0abcafc541acaee84f0f77d2ca0a 2024-12-05T00:41:39,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/16b2dfd46df34995ae8eabc6b6ab16fc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/16b2dfd46df34995ae8eabc6b6ab16fc 2024-12-05T00:41:39,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d536af276a6549768e33cdcdc966ded0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d536af276a6549768e33cdcdc966ded0 2024-12-05T00:41:39,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d4c8788e1f59419cb92ca3b8e6cdd4c6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/d4c8788e1f59419cb92ca3b8e6cdd4c6 2024-12-05T00:41:39,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9aa8930fd4b74b6e8f81552e49d3fd13 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/9aa8930fd4b74b6e8f81552e49d3fd13 2024-12-05T00:41:39,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/93c30de9024342649cc4f33d27809434 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/93c30de9024342649cc4f33d27809434 2024-12-05T00:41:39,403 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/recovered.edits/506.seqid, newMaxSeqId=506, maxSeqId=1 2024-12-05T00:41:39,404 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. 2024-12-05T00:41:39,405 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for d5061eee39269f2ef93c654f6e1034a0: Waiting for close lock at 1733359297977Running coprocessor pre-close hooks at 1733359297977Disabling compacts and flushes for region at 1733359297977Disabling writes for close at 1733359297977Obtaining lock to block concurrent updates at 1733359297977Preparing flush snapshotting stores in d5061eee39269f2ef93c654f6e1034a0 at 1733359297977Finished memstore snapshotting TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733359297978 (+1 ms)Flushing stores of TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0. at 1733359297978Flushing d5061eee39269f2ef93c654f6e1034a0/A: creating writer at 1733359297978Flushing d5061eee39269f2ef93c654f6e1034a0/A: appending metadata at 1733359297981 (+3 ms)Flushing d5061eee39269f2ef93c654f6e1034a0/A: closing flushed file at 1733359297981Flushing d5061eee39269f2ef93c654f6e1034a0/B: creating writer at 1733359298399 (+418 ms)Flushing d5061eee39269f2ef93c654f6e1034a0/B: appending metadata at 1733359298404 (+5 ms)Flushing d5061eee39269f2ef93c654f6e1034a0/B: closing flushed file at 1733359298404Flushing d5061eee39269f2ef93c654f6e1034a0/C: creating writer at 1733359298819 (+415 ms)Flushing d5061eee39269f2ef93c654f6e1034a0/C: appending metadata at 1733359298824 (+5 ms)Flushing d5061eee39269f2ef93c654f6e1034a0/C: closing flushed file at 1733359298824Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4da15dc1: reopening flushed file at 1733359299243 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5793a2da: reopening flushed file at 1733359299251 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d2c6237: reopening flushed file at 1733359299260 (+9 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d5061eee39269f2ef93c654f6e1034a0 in 1289ms, sequenceid=503, compaction requested=true at 1733359299267 (+7 ms)Writing region close event to WAL at 1733359299400 (+133 ms)Running coprocessor post-close hooks at 1733359299404 (+4 ms)Closed at 1733359299404 2024-12-05T00:41:39,407 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:39,408 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=d5061eee39269f2ef93c654f6e1034a0, regionState=CLOSED 2024-12-05T00:41:39,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:41:39,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T00:41:39,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure d5061eee39269f2ef93c654f6e1034a0, server=fea72ea5c4b6,42683,1733359267474 in 1.5910 sec 2024-12-05T00:41:39,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-05T00:41:39,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5061eee39269f2ef93c654f6e1034a0, UNASSIGN in 1.5990 sec 2024-12-05T00:41:39,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T00:41:39,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6080 sec 2024-12-05T00:41:39,421 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359299421"}]},"ts":"1733359299421"} 2024-12-05T00:41:39,423 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T00:41:39,423 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T00:41:39,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6240 sec 2024-12-05T00:41:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T00:41:39,952 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:39,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T00:41:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:39,960 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-05T00:41:39,962 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:39,967 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:39,970 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/recovered.edits] 2024-12-05T00:41:39,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/8bda1d894a494663aeaae61ff0b89ea0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/8bda1d894a494663aeaae61ff0b89ea0 2024-12-05T00:41:39,974 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/cd3041b2a1fb4318a1277fe27f2469bf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/cd3041b2a1fb4318a1277fe27f2469bf 2024-12-05T00:41:39,976 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/e82eaf179386488b949bf94d6c4ccb29 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/A/e82eaf179386488b949bf94d6c4ccb29 2024-12-05T00:41:39,978 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/05109376cfa7418d8eb99ea16b8f696d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/05109376cfa7418d8eb99ea16b8f696d 2024-12-05T00:41:39,979 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/46edcd81951b49299b93daa325a88354 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/46edcd81951b49299b93daa325a88354 2024-12-05T00:41:39,981 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77acd4dd654b4380b35603d3c434051c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/B/77acd4dd654b4380b35603d3c434051c 2024-12-05T00:41:39,984 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/062177a543544cf39bb90288e78b8cf8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/062177a543544cf39bb90288e78b8cf8 2024-12-05T00:41:39,985 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/48740871feb94436817d1aae323dbd6a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/48740871feb94436817d1aae323dbd6a 2024-12-05T00:41:39,987 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/928c4334b17f4b4090a87e5b34f0a289 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/C/928c4334b17f4b4090a87e5b34f0a289 2024-12-05T00:41:39,990 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/recovered.edits/506.seqid to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0/recovered.edits/506.seqid 2024-12-05T00:41:39,991 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/d5061eee39269f2ef93c654f6e1034a0 2024-12-05T00:41:39,991 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T00:41:39,996 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:40,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T00:41:40,004 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T00:41:40,035 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T00:41:40,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:40,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T00:41:40,037 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733359300037"}]},"ts":"9223372036854775807"} 2024-12-05T00:41:40,043 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T00:41:40,043 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d5061eee39269f2ef93c654f6e1034a0, NAME => 'TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T00:41:40,043 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T00:41:40,043 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733359300043"}]},"ts":"9223372036854775807"} 2024-12-05T00:41:40,047 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T00:41:40,048 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:40,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-12-05T00:41:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-05T00:41:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T00:41:40,073 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:40,086 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-866614165_22 at /127.0.0.1:44380 [Waiting for operation #155] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fea72ea5c4b6:42683-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1522171622_22 at /127.0.0.1:44088 [Waiting for operation #259] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 251) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9464 (was 10049) 2024-12-05T00:41:40,095 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=9464 2024-12-05T00:41:40,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:41:40,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:41:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:40,099 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:41:40,099 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:40,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-05T00:41:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T00:41:40,101 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:41:40,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741945_1121 (size=1061) 2024-12-05T00:41:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T00:41:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T00:41:40,509 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:41:40,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741946_1122 (size=53) 2024-12-05T00:41:40,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:40,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c39421bab8284c2f34cba4e4ea472def, disabling compactions & flushes 2024-12-05T00:41:40,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. after waiting 0 ms 2024-12-05T00:41:40,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c39421bab8284c2f34cba4e4ea472def: Waiting for close lock at 1733359300524Disabling compacts and flushes for region at 1733359300524Disabling writes for close at 1733359300524Writing region close event to WAL at 1733359300524Closed at 1733359300524 2024-12-05T00:41:40,526 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:41:40,527 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733359300526"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359300526"}]},"ts":"1733359300526"} 2024-12-05T00:41:40,530 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:41:40,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:41:40,532 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359300532"}]},"ts":"1733359300532"} 2024-12-05T00:41:40,535 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T00:41:40,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, ASSIGN}] 2024-12-05T00:41:40,537 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, ASSIGN 2024-12-05T00:41:40,538 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=false 2024-12-05T00:41:40,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:40,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, ASSIGN because future has completed 2024-12-05T00:41:40,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:41:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T00:41:40,849 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,849 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:41:40,850 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,850 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:40,850 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,850 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,852 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,854 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:40,856 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39421bab8284c2f34cba4e4ea472def columnFamilyName A 2024-12-05T00:41:40,856 DEBUG [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:40,857 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(327): Store=c39421bab8284c2f34cba4e4ea472def/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:40,857 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,859 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:40,859 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39421bab8284c2f34cba4e4ea472def columnFamilyName B 2024-12-05T00:41:40,859 DEBUG [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:40,860 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(327): Store=c39421bab8284c2f34cba4e4ea472def/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:40,872 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,873 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:40,873 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39421bab8284c2f34cba4e4ea472def columnFamilyName C 2024-12-05T00:41:40,873 DEBUG [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:40,874 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(327): Store=c39421bab8284c2f34cba4e4ea472def/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:40,874 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,874 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,875 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,875 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,876 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,876 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,876 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:41:40,877 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,879 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:41:40,880 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c39421bab8284c2f34cba4e4ea472def; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61008898, jitterRate=-0.09089657664299011}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:41:40,880 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:40,881 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c39421bab8284c2f34cba4e4ea472def: Running coprocessor pre-open hook at 1733359300850Writing region info on filesystem at 1733359300850Initializing all the Stores at 1733359300851 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359300851Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359300851Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359300852 (+1 ms)Cleaning up temporary data from old regions at 1733359300876 (+24 ms)Running coprocessor post-open hooks at 1733359300880 (+4 ms)Region opened successfully at 1733359300881 (+1 ms) 2024-12-05T00:41:40,882 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., pid=36, masterSystemTime=1733359300845 2024-12-05T00:41:40,884 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,884 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:40,884 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:40,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:41:40,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-05T00:41:40,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 in 194 msec 2024-12-05T00:41:40,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-05T00:41:40,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, ASSIGN in 353 msec 2024-12-05T00:41:40,890 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:41:40,890 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359300890"}]},"ts":"1733359300890"} 2024-12-05T00:41:40,891 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T00:41:40,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:41:40,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 795 msec 2024-12-05T00:41:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T00:41:41,242 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:41,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e732f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:41,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:41,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:41,244 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d4f81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:41,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79906bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:41,247 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:41,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:41,250 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:41:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:41:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e27d07 2024-12-05T00:41:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:41:41,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:41:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:41:41,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:41:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T00:41:41,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741947_1123 (size=1097) 2024-12-05T00:41:41,679 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T00:41:41,679 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T00:41:41,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:41:41,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, REOPEN/MOVE}] 2024-12-05T00:41:41,694 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, REOPEN/MOVE 2024-12-05T00:41:41,695 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:41,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, REOPEN/MOVE because future has completed 2024-12-05T00:41:41,697 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:41:41,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:41:41,852 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:41,852 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:41:41,852 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing c39421bab8284c2f34cba4e4ea472def, disabling compactions & flushes 2024-12-05T00:41:41,852 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:41,852 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:41,853 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. after waiting 0 ms 2024-12-05T00:41:41,853 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:41,862 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T00:41:41,863 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:41,863 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for c39421bab8284c2f34cba4e4ea472def: Waiting for close lock at 1733359301852Running coprocessor pre-close hooks at 1733359301852Disabling compacts and flushes for region at 1733359301852Disabling writes for close at 1733359301853 (+1 ms)Writing region close event to WAL at 1733359301854 (+1 ms)Running coprocessor post-close hooks at 1733359301863 (+9 ms)Closed at 1733359301863 2024-12-05T00:41:41,863 WARN [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: c39421bab8284c2f34cba4e4ea472def to self. 2024-12-05T00:41:41,865 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:41,866 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=CLOSED 2024-12-05T00:41:41,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:41:41,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-05T00:41:41,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 in 171 msec 2024-12-05T00:41:41,870 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, REOPEN/MOVE; state=CLOSED, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=true 2024-12-05T00:41:42,021 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, REOPEN/MOVE because future has completed 2024-12-05T00:41:42,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:41:42,182 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,182 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:41:42,183 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,183 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:41:42,183 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,183 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,192 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,193 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:42,199 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39421bab8284c2f34cba4e4ea472def columnFamilyName A 2024-12-05T00:41:42,200 DEBUG [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:42,201 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(327): Store=c39421bab8284c2f34cba4e4ea472def/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:42,201 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,202 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:42,202 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39421bab8284c2f34cba4e4ea472def columnFamilyName B 2024-12-05T00:41:42,202 DEBUG [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:42,203 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(327): Store=c39421bab8284c2f34cba4e4ea472def/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:42,203 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,204 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:41:42,204 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39421bab8284c2f34cba4e4ea472def columnFamilyName C 2024-12-05T00:41:42,204 DEBUG [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:42,204 INFO [StoreOpener-c39421bab8284c2f34cba4e4ea472def-1 {}] regionserver.HStore(327): Store=c39421bab8284c2f34cba4e4ea472def/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:41:42,204 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,204 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,205 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,206 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,207 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,207 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,207 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:41:42,209 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,209 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c39421bab8284c2f34cba4e4ea472def; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71949400, jitterRate=0.07212960720062256}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:41:42,209 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,210 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c39421bab8284c2f34cba4e4ea472def: Running coprocessor pre-open hook at 1733359302184Writing region info on filesystem at 1733359302184Initializing all the Stores at 1733359302186 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359302186Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359302186Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359302186Cleaning up temporary data from old regions at 1733359302207 (+21 ms)Running coprocessor post-open hooks at 1733359302209 (+2 ms)Region opened successfully at 1733359302210 (+1 ms) 2024-12-05T00:41:42,211 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., pid=41, masterSystemTime=1733359302177 2024-12-05T00:41:42,213 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,213 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,213 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=OPEN, openSeqNum=5, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:41:42,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-05T00:41:42,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 in 191 msec 2024-12-05T00:41:42,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-05T00:41:42,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, REOPEN/MOVE in 524 msec 2024-12-05T00:41:42,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-05T00:41:42,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-12-05T00:41:42,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-12-05T00:41:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-05T00:41:42,227 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:42,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4716031f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,229 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429fa85b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,230 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1da99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d3ba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,233 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f16f471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ccd683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,236 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168ce763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebab769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47250f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,240 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f4a6b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,241 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2555a8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,244 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69dc99c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f48f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed913a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,248 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5bb495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c9a6dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49519de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d06103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,252 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c58b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16801009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,254 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b621a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:41:42,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:41:42,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:41:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:41:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:41:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c9b05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:41:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:41:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:41:42,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:41:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3177e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:41:42,261 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,261 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,261 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,261 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,261 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,262 DEBUG [Thread-634 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,262 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,263 DEBUG [Thread-634 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,263 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:42,263 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-05T00:41:42,264 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,264 DEBUG [Thread-636 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,265 DEBUG [Thread-636 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T00:41:42,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,267 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:41:42,268 DEBUG [Thread-623 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:41:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,268 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:42,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:42,268 DEBUG [Thread-623 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:41:42,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:41:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:41:42,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:41:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359362323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359362323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359362325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205adab03c701624e97a2b4b01a69f7aa3d_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359302283/Put/seqid=0 2024-12-05T00:41:42,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359302307, max timestamp=1733359302335Num uniques 859; 2024-12-05T00:41:42,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:41:42,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359302307, max timestamp=1733359302335Num uniques 859; 2024-12-05T00:41:42,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:41:42,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733359302307, max timestamp=1733359302335Num uniques 859; 2024-12-05T00:41:42,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:41:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741948_1124 (size=12154) 2024-12-05T00:41:42,361 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:42,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205adab03c701624e97a2b4b01a69f7aa3d_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205adab03c701624e97a2b4b01a69f7aa3d_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T00:41:42,372 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/492e62bb6d1a414fb6e9a6f61d2b8e7a, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:42,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/492e62bb6d1a414fb6e9a6f61d2b8e7a is 175, key is test_row_0/A:col10/1733359302283/Put/seqid=0 2024-12-05T00:41:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741949_1125 (size=30955) 2024-12-05T00:41:42,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/492e62bb6d1a414fb6e9a6f61d2b8e7a 2024-12-05T00:41:42,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359362395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359362402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/11b75747dd15402b8a1a340c3112937c is 50, key is test_row_0/B:col10/1733359302283/Put/seqid=0 2024-12-05T00:41:42,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T00:41:42,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:42,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:42,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359362442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359362442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359362442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741950_1126 (size=12001) 2024-12-05T00:41:42,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/11b75747dd15402b8a1a340c3112937c 2024-12-05T00:41:42,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/3e96a88ef8b14485aa9dd2676052f93a is 50, key is test_row_0/C:col10/1733359302283/Put/seqid=0 2024-12-05T00:41:42,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741951_1127 (size=12001) 2024-12-05T00:41:42,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/3e96a88ef8b14485aa9dd2676052f93a 2024-12-05T00:41:42,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/492e62bb6d1a414fb6e9a6f61d2b8e7a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a 2024-12-05T00:41:42,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359362512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a, entries=150, sequenceid=15, filesize=30.2 K 2024-12-05T00:41:42,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/11b75747dd15402b8a1a340c3112937c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/11b75747dd15402b8a1a340c3112937c 2024-12-05T00:41:42,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359362522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/11b75747dd15402b8a1a340c3112937c, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T00:41:42,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/3e96a88ef8b14485aa9dd2676052f93a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/3e96a88ef8b14485aa9dd2676052f93a 2024-12-05T00:41:42,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/3e96a88ef8b14485aa9dd2676052f93a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T00:41:42,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c39421bab8284c2f34cba4e4ea472def in 264ms, sequenceid=15, compaction requested=false 2024-12-05T00:41:42,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:42,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T00:41:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:42,577 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-05T00:41:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T00:41:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cca7f5a3af414737aaccedef91990c75_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359302307/Put/seqid=0 2024-12-05T00:41:42,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:42,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359362669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741952_1128 (size=12154) 2024-12-05T00:41:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359362671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359362673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359362722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359362742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359362781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359362782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359362782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T00:41:42,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359362992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359362992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359363002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359363031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359363052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:43,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cca7f5a3af414737aaccedef91990c75_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cca7f5a3af414737aaccedef91990c75_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:43,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/72b6401806d74453b175405646df19ad, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:43,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/72b6401806d74453b175405646df19ad is 175, key is test_row_0/A:col10/1733359302307/Put/seqid=0 2024-12-05T00:41:43,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741953_1129 (size=30955) 2024-12-05T00:41:43,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/72b6401806d74453b175405646df19ad 2024-12-05T00:41:43,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/b01d3b94170a422d924a44f2733a7976 is 50, key is test_row_0/B:col10/1733359302307/Put/seqid=0 2024-12-05T00:41:43,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741954_1130 (size=12001) 2024-12-05T00:41:43,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/b01d3b94170a422d924a44f2733a7976 2024-12-05T00:41:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/dac94c2b33504cab9baaefa568e2b94e is 50, key is test_row_0/C:col10/1733359302307/Put/seqid=0 2024-12-05T00:41:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741955_1131 (size=12001) 2024-12-05T00:41:43,153 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/dac94c2b33504cab9baaefa568e2b94e 2024-12-05T00:41:43,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/72b6401806d74453b175405646df19ad as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad 2024-12-05T00:41:43,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad, entries=150, sequenceid=49, filesize=30.2 K 2024-12-05T00:41:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/b01d3b94170a422d924a44f2733a7976 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/b01d3b94170a422d924a44f2733a7976 2024-12-05T00:41:43,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/b01d3b94170a422d924a44f2733a7976, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T00:41:43,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/dac94c2b33504cab9baaefa568e2b94e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/dac94c2b33504cab9baaefa568e2b94e 2024-12-05T00:41:43,186 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/dac94c2b33504cab9baaefa568e2b94e, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T00:41:43,188 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for c39421bab8284c2f34cba4e4ea472def in 611ms, sequenceid=49, compaction requested=false 2024-12-05T00:41:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-05T00:41:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-05T00:41:43,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T00:41:43,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 922 msec 2024-12-05T00:41:43,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 929 msec 2024-12-05T00:41:43,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:41:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:43,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205022e6b2098484448817ace39d0442c3c_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741956_1132 (size=14594) 2024-12-05T00:41:43,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359363346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359363348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359363352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205022e6b2098484448817ace39d0442c3c_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205022e6b2098484448817ace39d0442c3c_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/31c8c107ab3643bc903a5f5d7af6946f, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:43,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/31c8c107ab3643bc903a5f5d7af6946f is 175, key is test_row_0/A:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741957_1133 (size=39549) 2024-12-05T00:41:43,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/31c8c107ab3643bc903a5f5d7af6946f 2024-12-05T00:41:43,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/e82163e373c4428ebdece49f94903e30 is 50, key is test_row_0/B:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741958_1134 (size=12001) 2024-12-05T00:41:43,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/e82163e373c4428ebdece49f94903e30 2024-12-05T00:41:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T00:41:43,403 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:43,403 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-05T00:41:43,407 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:43,409 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:43,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:43,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/77f8f67bae104ee4901a3304c43bfd4b is 50, key is test_row_0/C:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741959_1135 (size=12001) 2024-12-05T00:41:43,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/77f8f67bae104ee4901a3304c43bfd4b 2024-12-05T00:41:43,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/31c8c107ab3643bc903a5f5d7af6946f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f 2024-12-05T00:41:43,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359363462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359363462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359363463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f, entries=200, sequenceid=62, filesize=38.6 K 2024-12-05T00:41:43,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/e82163e373c4428ebdece49f94903e30 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/e82163e373c4428ebdece49f94903e30 2024-12-05T00:41:43,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/e82163e373c4428ebdece49f94903e30, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T00:41:43,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/77f8f67bae104ee4901a3304c43bfd4b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/77f8f67bae104ee4901a3304c43bfd4b 2024-12-05T00:41:43,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/77f8f67bae104ee4901a3304c43bfd4b, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T00:41:43,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c39421bab8284c2f34cba4e4ea472def in 178ms, sequenceid=62, compaction requested=true 2024-12-05T00:41:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:43,483 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:43,484 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:43,485 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:43,485 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:43,486 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:41:43,486 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:41:43,486 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,486 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,486 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=99.1 K 2024-12-05T00:41:43,486 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/11b75747dd15402b8a1a340c3112937c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/b01d3b94170a422d924a44f2733a7976, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/e82163e373c4428ebdece49f94903e30] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=35.2 K 2024-12-05T00:41:43,486 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,486 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f] 2024-12-05T00:41:43,487 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b75747dd15402b8a1a340c3112937c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359302277 2024-12-05T00:41:43,487 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492e62bb6d1a414fb6e9a6f61d2b8e7a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359302277 2024-12-05T00:41:43,488 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72b6401806d74453b175405646df19ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733359302307 2024-12-05T00:41:43,488 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b01d3b94170a422d924a44f2733a7976, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733359302307 2024-12-05T00:41:43,488 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31c8c107ab3643bc903a5f5d7af6946f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359302664 2024-12-05T00:41:43,488 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e82163e373c4428ebdece49f94903e30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359302664 2024-12-05T00:41:43,498 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:43,499 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:43,500 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/a301201d2bf34952bd208cfe849af291 is 50, key is test_row_0/B:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,505 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412055c631592bf5048a79365f1fc09106005_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:43,510 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412055c631592bf5048a79365f1fc09106005_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:43,511 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055c631592bf5048a79365f1fc09106005_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:43,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741960_1136 (size=12104) 2024-12-05T00:41:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:43,527 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/a301201d2bf34952bd208cfe849af291 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a301201d2bf34952bd208cfe849af291 2024-12-05T00:41:43,534 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into a301201d2bf34952bd208cfe849af291(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:43,534 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:43,534 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359303484; duration=0sec 2024-12-05T00:41:43,535 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:43,535 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:41:43,535 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:43,536 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:43,536 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:41:43,536 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,537 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/3e96a88ef8b14485aa9dd2676052f93a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/dac94c2b33504cab9baaefa568e2b94e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/77f8f67bae104ee4901a3304c43bfd4b] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=35.2 K 2024-12-05T00:41:43,538 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e96a88ef8b14485aa9dd2676052f93a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359302277 2024-12-05T00:41:43,539 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting dac94c2b33504cab9baaefa568e2b94e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733359302307 2024-12-05T00:41:43,539 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f8f67bae104ee4901a3304c43bfd4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359302664 2024-12-05T00:41:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:43,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:43,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741961_1137 (size=4469) 2024-12-05T00:41:43,548 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#118 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:43,548 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/525259769ab0475a8ee2e97e39c4a803 is 175, key is test_row_0/A:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,556 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#119 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:43,556 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/0a53c7af1e4d4091930d31ab14597461 is 50, key is test_row_0/C:col10/1733359302664/Put/seqid=0 2024-12-05T00:41:43,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741962_1138 (size=31058) 2024-12-05T00:41:43,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053128122070d442a9a47fa5a038ec0929_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359303350/Put/seqid=0 2024-12-05T00:41:43,567 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/525259769ab0475a8ee2e97e39c4a803 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/525259769ab0475a8ee2e97e39c4a803 2024-12-05T00:41:43,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741963_1139 (size=12104) 2024-12-05T00:41:43,574 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into 525259769ab0475a8ee2e97e39c4a803(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:43,574 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:43,574 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359303483; duration=0sec 2024-12-05T00:41:43,574 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:43,574 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:41:43,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T00:41:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359363576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,582 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/0a53c7af1e4d4091930d31ab14597461 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0a53c7af1e4d4091930d31ab14597461 2024-12-05T00:41:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359363581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,589 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 0a53c7af1e4d4091930d31ab14597461(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:43,589 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:43,589 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359303484; duration=0sec 2024-12-05T00:41:43,589 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:43,589 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:41:43,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741964_1140 (size=14594) 2024-12-05T00:41:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359363672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359363672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359363672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359363682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359363692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:43,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T00:41:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T00:41:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:43,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359363892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359363902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359363982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359363982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359363982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:44,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053128122070d442a9a47fa5a038ec0929_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053128122070d442a9a47fa5a038ec0929_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:44,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/29d0769ed9194b76b34131947a16644d, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:44,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/29d0769ed9194b76b34131947a16644d is 175, key is test_row_0/A:col10/1733359303350/Put/seqid=0 2024-12-05T00:41:44,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741965_1141 (size=39549) 2024-12-05T00:41:44,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/29d0769ed9194b76b34131947a16644d 2024-12-05T00:41:44,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/c28ed701b717445ea285dfccae7d950b is 50, key is test_row_0/B:col10/1733359303350/Put/seqid=0 2024-12-05T00:41:44,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T00:41:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:44,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:44,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741966_1142 (size=12001) 2024-12-05T00:41:44,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/c28ed701b717445ea285dfccae7d950b 2024-12-05T00:41:44,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/4ccd832a80b1473bb36187d05060ada6 is 50, key is test_row_0/C:col10/1733359303350/Put/seqid=0 2024-12-05T00:41:44,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741967_1143 (size=12001) 2024-12-05T00:41:44,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/4ccd832a80b1473bb36187d05060ada6 2024-12-05T00:41:44,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/29d0769ed9194b76b34131947a16644d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d 2024-12-05T00:41:44,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d, entries=200, sequenceid=83, filesize=38.6 K 2024-12-05T00:41:44,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/c28ed701b717445ea285dfccae7d950b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c28ed701b717445ea285dfccae7d950b 2024-12-05T00:41:44,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c28ed701b717445ea285dfccae7d950b, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T00:41:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/4ccd832a80b1473bb36187d05060ada6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/4ccd832a80b1473bb36187d05060ada6 2024-12-05T00:41:44,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/4ccd832a80b1473bb36187d05060ada6, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T00:41:44,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c39421bab8284c2f34cba4e4ea472def in 572ms, sequenceid=83, compaction requested=false 2024-12-05T00:41:44,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:44,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T00:41:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:44,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:41:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054fede699d970498ba52a80987acfd8ee_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359303573/Put/seqid=0 2024-12-05T00:41:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:44,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741968_1144 (size=12154) 2024-12-05T00:41:44,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359364241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359364245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359364352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359364352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359364492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359364492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359364492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359364562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359364562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:44,624 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054fede699d970498ba52a80987acfd8ee_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054fede699d970498ba52a80987acfd8ee_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:44,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/57505246e04e49c19106ffc0954e9afa, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:44,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/57505246e04e49c19106ffc0954e9afa is 175, key is test_row_0/A:col10/1733359303573/Put/seqid=0 2024-12-05T00:41:44,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741969_1145 (size=30955) 2024-12-05T00:41:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359364871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359364871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:44,944 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:41:45,030 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/57505246e04e49c19106ffc0954e9afa 2024-12-05T00:41:45,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/27ad0542cf2142c78def7435d9f9b21b is 50, key is test_row_0/B:col10/1733359303573/Put/seqid=0 2024-12-05T00:41:45,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741970_1146 (size=12001) 2024-12-05T00:41:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359365382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359365382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:45,443 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/27ad0542cf2142c78def7435d9f9b21b 2024-12-05T00:41:45,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/f2846c5c8c0c45bc8715390c7cb83c43 is 50, key is test_row_0/C:col10/1733359303573/Put/seqid=0 2024-12-05T00:41:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741971_1147 (size=12001) 2024-12-05T00:41:45,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359365502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:45,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359365502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:45,505 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:45,505 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:45,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359365511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:45,515 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:45,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/f2846c5c8c0c45bc8715390c7cb83c43 2024-12-05T00:41:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/57505246e04e49c19106ffc0954e9afa as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa 2024-12-05T00:41:45,873 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa, entries=150, sequenceid=97, filesize=30.2 K 2024-12-05T00:41:45,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/27ad0542cf2142c78def7435d9f9b21b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/27ad0542cf2142c78def7435d9f9b21b 2024-12-05T00:41:45,878 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/27ad0542cf2142c78def7435d9f9b21b, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:41:45,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/f2846c5c8c0c45bc8715390c7cb83c43 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f2846c5c8c0c45bc8715390c7cb83c43 2024-12-05T00:41:45,886 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f2846c5c8c0c45bc8715390c7cb83c43, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:41:45,888 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c39421bab8284c2f34cba4e4ea472def in 1695ms, sequenceid=97, compaction requested=true 2024-12-05T00:41:45,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:45,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:45,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-05T00:41:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-05T00:41:45,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T00:41:45,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4810 sec 2024-12-05T00:41:45,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4890 sec 2024-12-05T00:41:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:46,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-05T00:41:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:46,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:46,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:41:46,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a55f07d39364694b0f118bd71688468_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:46,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741972_1148 (size=17034) 2024-12-05T00:41:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359366424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359366428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359366531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:46,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359366541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359366742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359366752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:46,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:46,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a55f07d39364694b0f118bd71688468_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a55f07d39364694b0f118bd71688468_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:46,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/572b83511a1045e29af2987e12d1f0e9, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:46,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/572b83511a1045e29af2987e12d1f0e9 is 175, key is test_row_0/A:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:46,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741973_1149 (size=48139) 2024-12-05T00:41:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359367051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359367061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/572b83511a1045e29af2987e12d1f0e9 2024-12-05T00:41:47,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/0da1f8ccb8d340d9820edefe75b78e59 is 50, key is test_row_0/B:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:47,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741974_1150 (size=12001) 2024-12-05T00:41:47,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359367522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,523 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359367532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359367532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,535 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,535 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359367561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359367572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T00:41:47,582 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:47,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-05T00:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T00:41:47,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:47,585 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:47,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:47,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/0da1f8ccb8d340d9820edefe75b78e59 2024-12-05T00:41:47,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/57e0205dfaec44e7a43e8db1af5c544f is 50, key is test_row_0/C:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741975_1151 (size=12001) 2024-12-05T00:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T00:41:47,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T00:41:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:47,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:47,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T00:41:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:47,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T00:41:48,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T00:41:48,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:48,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/57e0205dfaec44e7a43e8db1af5c544f 2024-12-05T00:41:48,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/572b83511a1045e29af2987e12d1f0e9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9 2024-12-05T00:41:48,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9, entries=250, sequenceid=119, filesize=47.0 K 2024-12-05T00:41:48,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/0da1f8ccb8d340d9820edefe75b78e59 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0da1f8ccb8d340d9820edefe75b78e59 2024-12-05T00:41:48,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0da1f8ccb8d340d9820edefe75b78e59, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T00:41:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/57e0205dfaec44e7a43e8db1af5c544f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/57e0205dfaec44e7a43e8db1af5c544f 2024-12-05T00:41:48,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/57e0205dfaec44e7a43e8db1af5c544f, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T00:41:48,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for c39421bab8284c2f34cba4e4ea472def in 1683ms, sequenceid=119, compaction requested=true 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:48,087 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:48,087 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:48,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:48,089 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:48,089 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:48,089 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:41:48,089 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:41:48,089 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,089 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,089 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a301201d2bf34952bd208cfe849af291, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c28ed701b717445ea285dfccae7d950b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/27ad0542cf2142c78def7435d9f9b21b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0da1f8ccb8d340d9820edefe75b78e59] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=47.0 K 2024-12-05T00:41:48,089 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/525259769ab0475a8ee2e97e39c4a803, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=146.2 K 2024-12-05T00:41:48,089 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,089 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/525259769ab0475a8ee2e97e39c4a803, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9] 2024-12-05T00:41:48,090 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a301201d2bf34952bd208cfe849af291, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359302664 2024-12-05T00:41:48,090 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525259769ab0475a8ee2e97e39c4a803, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359302664 2024-12-05T00:41:48,090 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c28ed701b717445ea285dfccae7d950b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733359303342 2024-12-05T00:41:48,090 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d0769ed9194b76b34131947a16644d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733359303340 2024-12-05T00:41:48,090 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 27ad0542cf2142c78def7435d9f9b21b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359303562 2024-12-05T00:41:48,091 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57505246e04e49c19106ffc0954e9afa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359303562 2024-12-05T00:41:48,091 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da1f8ccb8d340d9820edefe75b78e59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733359304239 2024-12-05T00:41:48,091 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572b83511a1045e29af2987e12d1f0e9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733359304236 2024-12-05T00:41:48,101 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:48,107 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412054956a9bb0737400984a5163bb78ba75e_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:48,107 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:48,108 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/78afc9f867de41999ed8daec70e82edb is 50, key is test_row_0/B:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:48,110 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412054956a9bb0737400984a5163bb78ba75e_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:48,110 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054956a9bb0737400984a5163bb78ba75e_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:48,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741976_1152 (size=4469) 2024-12-05T00:41:48,126 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#129 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:48,127 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/d86cc77a7ca14ca9a9bb5ca68bb14372 is 175, key is test_row_0/A:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:48,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741977_1153 (size=12241) 2024-12-05T00:41:48,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741978_1154 (size=31195) 2024-12-05T00:41:48,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T00:41:48,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,196 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:41:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:48,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051d555f24de634ff1b90a8c9064553609_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359306421/Put/seqid=0 2024-12-05T00:41:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741979_1155 (size=12154) 2024-12-05T00:41:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T00:41:48,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051d555f24de634ff1b90a8c9064553609_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051d555f24de634ff1b90a8c9064553609_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:48,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/13161be3336845a0b676752af1d62e41, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:48,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/13161be3336845a0b676752af1d62e41 is 175, key is test_row_0/A:col10/1733359306421/Put/seqid=0 2024-12-05T00:41:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741980_1156 (size=30955) 2024-12-05T00:41:48,544 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/78afc9f867de41999ed8daec70e82edb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/78afc9f867de41999ed8daec70e82edb 2024-12-05T00:41:48,548 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/d86cc77a7ca14ca9a9bb5ca68bb14372 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d86cc77a7ca14ca9a9bb5ca68bb14372 2024-12-05T00:41:48,550 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into 78afc9f867de41999ed8daec70e82edb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:48,550 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:48,550 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=12, startTime=1733359308087; duration=0sec 2024-12-05T00:41:48,550 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:48,550 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:41:48,550 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:41:48,552 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:41:48,552 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:41:48,552 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:48,552 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0a53c7af1e4d4091930d31ab14597461, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/4ccd832a80b1473bb36187d05060ada6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f2846c5c8c0c45bc8715390c7cb83c43, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/57e0205dfaec44e7a43e8db1af5c544f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=47.0 K 2024-12-05T00:41:48,553 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a53c7af1e4d4091930d31ab14597461, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359302664 2024-12-05T00:41:48,553 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ccd832a80b1473bb36187d05060ada6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733359303342 2024-12-05T00:41:48,553 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting f2846c5c8c0c45bc8715390c7cb83c43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359303562 2024-12-05T00:41:48,554 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e0205dfaec44e7a43e8db1af5c544f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733359304239 2024-12-05T00:41:48,556 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into d86cc77a7ca14ca9a9bb5ca68bb14372(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:48,556 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:48,556 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=12, startTime=1733359308087; duration=0sec 2024-12-05T00:41:48,556 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:48,556 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:41:48,562 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:48,563 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/0fbadf62836c49a99423b4ca0f588214 is 50, key is test_row_0/C:col10/1733359306402/Put/seqid=0 2024-12-05T00:41:48,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741981_1157 (size=12241) 2024-12-05T00:41:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:48,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:48,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359308585, max timestamp=1733359308622Num uniques 859; 2024-12-05T00:41:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359368624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:48,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359308585, max timestamp=1733359308622Num uniques 859; 2024-12-05T00:41:48,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:48,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:48,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/13161be3336845a0b676752af1d62e41 2024-12-05T00:41:48,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733359308585, max timestamp=1733359308622Num uniques 859; 2024-12-05T00:41:48,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T00:41:48,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/48a6926e1a9a402d9fc165be6baddfde is 50, key is test_row_0/B:col10/1733359306421/Put/seqid=0 2024-12-05T00:41:48,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741982_1158 (size=12001) 2024-12-05T00:41:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T00:41:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359368728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359368731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:48,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359368832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:48,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:48,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359368942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:48,982 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/0fbadf62836c49a99423b4ca0f588214 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0fbadf62836c49a99423b4ca0f588214 2024-12-05T00:41:48,987 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 0fbadf62836c49a99423b4ca0f588214(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:48,987 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:48,987 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=12, startTime=1733359308087; duration=0sec 2024-12-05T00:41:48,987 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:48,987 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:41:49,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/48a6926e1a9a402d9fc165be6baddfde 2024-12-05T00:41:49,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359369041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/6e6b3b859fe2489fad829c7e1fc3f049 is 50, key is test_row_0/C:col10/1733359306421/Put/seqid=0 2024-12-05T00:41:49,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741983_1159 (size=12001) 2024-12-05T00:41:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359369252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:49,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359369352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:49,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:49,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/6e6b3b859fe2489fad829c7e1fc3f049 2024-12-05T00:41:49,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/13161be3336845a0b676752af1d62e41 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41 2024-12-05T00:41:49,465 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41, entries=150, sequenceid=129, filesize=30.2 K 2024-12-05T00:41:49,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/48a6926e1a9a402d9fc165be6baddfde as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/48a6926e1a9a402d9fc165be6baddfde 2024-12-05T00:41:49,471 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/48a6926e1a9a402d9fc165be6baddfde, entries=150, sequenceid=129, filesize=11.7 K 2024-12-05T00:41:49,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/6e6b3b859fe2489fad829c7e1fc3f049 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6e6b3b859fe2489fad829c7e1fc3f049 2024-12-05T00:41:49,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6e6b3b859fe2489fad829c7e1fc3f049, entries=150, sequenceid=129, filesize=11.7 K 2024-12-05T00:41:49,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for c39421bab8284c2f34cba4e4ea472def in 1285ms, sequenceid=129, compaction requested=false 2024-12-05T00:41:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-05T00:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-05T00:41:49,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-05T00:41:49,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-12-05T00:41:49,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9020 sec 2024-12-05T00:41:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T00:41:49,732 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:49,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-05T00:41:49,734 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T00:41:49,734 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:49,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:49,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-12-05T00:41:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T00:41:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T00:41:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T00:41:49,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205007c7099e493465d88eb69dac374321c_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:49,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741984_1160 (size=14794) 2024-12-05T00:41:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359369804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T00:41:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359369862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:49,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:49,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:49,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359369911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:50,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T00:41:50,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359370121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:50,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:50,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:50,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205007c7099e493465d88eb69dac374321c_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205007c7099e493465d88eb69dac374321c_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:50,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/642636cb5d8049d28596c76ec3b9fd2c, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:50,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/642636cb5d8049d28596c76ec3b9fd2c is 175, key is test_row_0/A:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741985_1161 (size=39749) 2024-12-05T00:41:50,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/642636cb5d8049d28596c76ec3b9fd2c 2024-12-05T00:41:50,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/ddac260b332b4a70b9411d7f15d87844 is 50, key is test_row_0/B:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:50,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741986_1162 (size=12151) 2024-12-05T00:41:50,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T00:41:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359370432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:50,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/ddac260b332b4a70b9411d7f15d87844 2024-12-05T00:41:50,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/60e75b2d7d9447d7b6a1abffff068ee4 is 50, key is test_row_0/C:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:50,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741987_1163 (size=12151) 2024-12-05T00:41:50,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T00:41:50,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359370871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:50,873 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2145 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:50,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359370942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:50,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:50,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:50,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:51,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/60e75b2d7d9447d7b6a1abffff068ee4 2024-12-05T00:41:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/642636cb5d8049d28596c76ec3b9fd2c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c 2024-12-05T00:41:51,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c, entries=200, sequenceid=170, filesize=38.8 K 2024-12-05T00:41:51,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/ddac260b332b4a70b9411d7f15d87844 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ddac260b332b4a70b9411d7f15d87844 2024-12-05T00:41:51,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ddac260b332b4a70b9411d7f15d87844, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T00:41:51,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/60e75b2d7d9447d7b6a1abffff068ee4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/60e75b2d7d9447d7b6a1abffff068ee4 2024-12-05T00:41:51,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/60e75b2d7d9447d7b6a1abffff068ee4, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T00:41:51,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~404.65 KB/414360, currentSize=40.25 KB/41220 for c39421bab8284c2f34cba4e4ea472def in 1300ms, sequenceid=170, compaction requested=true 2024-12-05T00:41:51,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:51,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:51,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:51,065 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:51,066 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:51,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:51,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:51,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:51,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:51,067 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:51,067 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:41:51,067 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:51,067 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d86cc77a7ca14ca9a9bb5ca68bb14372, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=99.5 K 2024-12-05T00:41:51,067 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:51,067 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d86cc77a7ca14ca9a9bb5ca68bb14372, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c] 2024-12-05T00:41:51,068 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:51,068 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:41:51,068 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:51,068 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/78afc9f867de41999ed8daec70e82edb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/48a6926e1a9a402d9fc165be6baddfde, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ddac260b332b4a70b9411d7f15d87844] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=35.5 K 2024-12-05T00:41:51,068 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d86cc77a7ca14ca9a9bb5ca68bb14372, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733359304239 2024-12-05T00:41:51,068 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 78afc9f867de41999ed8daec70e82edb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733359304239 2024-12-05T00:41:51,069 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13161be3336845a0b676752af1d62e41, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733359306421 2024-12-05T00:41:51,069 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a6926e1a9a402d9fc165be6baddfde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733359306421 2024-12-05T00:41:51,069 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642636cb5d8049d28596c76ec3b9fd2c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359308711 2024-12-05T00:41:51,069 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ddac260b332b4a70b9411d7f15d87844, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359308711 2024-12-05T00:41:51,090 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:51,091 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/60037a2ab2534d12ba5696b3d047f310 is 50, key is test_row_0/B:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:51,093 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:51,109 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205f1c8cf6fe404455486294323bb82cf71_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:51,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741988_1164 (size=12493) 2024-12-05T00:41:51,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T00:41:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:51,113 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T00:41:51,113 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205f1c8cf6fe404455486294323bb82cf71_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:51,113 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f1c8cf6fe404455486294323bb82cf71_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:51,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:51,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741989_1165 (size=4469) 2024-12-05T00:41:51,146 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#139 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:51,147 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/10324e5db35844f7b6af16aa6107b1ff is 175, key is test_row_0/A:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:51,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056ec831ed4f6f4c93ba6641d355eafcb6_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359309789/Put/seqid=0 2024-12-05T00:41:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741991_1167 (size=12304) 2024-12-05T00:41:51,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741990_1166 (size=31447) 2024-12-05T00:41:51,190 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056ec831ed4f6f4c93ba6641d355eafcb6_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056ec831ed4f6f4c93ba6641d355eafcb6_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:51,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/b427986c5d604defb2c49dd864aa19fd, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:51,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/b427986c5d604defb2c49dd864aa19fd is 175, key is test_row_0/A:col10/1733359309789/Put/seqid=0 2024-12-05T00:41:51,199 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/10324e5db35844f7b6af16aa6107b1ff as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/10324e5db35844f7b6af16aa6107b1ff 2024-12-05T00:41:51,210 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into 10324e5db35844f7b6af16aa6107b1ff(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:51,210 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:51,210 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359311065; duration=0sec 2024-12-05T00:41:51,210 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:51,210 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:41:51,210 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:51,211 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:51,211 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:41:51,211 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:51,211 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0fbadf62836c49a99423b4ca0f588214, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6e6b3b859fe2489fad829c7e1fc3f049, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/60e75b2d7d9447d7b6a1abffff068ee4] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=35.5 K 2024-12-05T00:41:51,213 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fbadf62836c49a99423b4ca0f588214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733359304239 2024-12-05T00:41:51,214 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6b3b859fe2489fad829c7e1fc3f049, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733359306421 2024-12-05T00:41:51,215 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60e75b2d7d9447d7b6a1abffff068ee4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359308711 2024-12-05T00:41:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741992_1168 (size=31105) 2024-12-05T00:41:51,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/b427986c5d604defb2c49dd864aa19fd 2024-12-05T00:41:51,240 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:51,240 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/23f10e0b09334629b10c31474bcfe1ad is 50, key is test_row_0/C:col10/1733359309763/Put/seqid=0 2024-12-05T00:41:51,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/9728828e37c34aabb60da22da7b5cd3d is 50, key is test_row_0/B:col10/1733359309789/Put/seqid=0 2024-12-05T00:41:51,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741993_1169 (size=12151) 2024-12-05T00:41:51,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/9728828e37c34aabb60da22da7b5cd3d 2024-12-05T00:41:51,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c is 50, key is test_row_0/C:col10/1733359309789/Put/seqid=0 2024-12-05T00:41:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741994_1170 (size=12493) 2024-12-05T00:41:51,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741995_1171 (size=12151) 2024-12-05T00:41:51,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c 2024-12-05T00:41:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/b427986c5d604defb2c49dd864aa19fd as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd 2024-12-05T00:41:51,372 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd, entries=150, sequenceid=179, filesize=30.4 K 2024-12-05T00:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/9728828e37c34aabb60da22da7b5cd3d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/9728828e37c34aabb60da22da7b5cd3d 2024-12-05T00:41:51,380 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/9728828e37c34aabb60da22da7b5cd3d, entries=150, sequenceid=179, filesize=11.9 K 2024-12-05T00:41:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c 2024-12-05T00:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,393 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c, entries=150, sequenceid=179, filesize=11.9 K 2024-12-05T00:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,395 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for c39421bab8284c2f34cba4e4ea472def in 282ms, sequenceid=179, compaction requested=false 2024-12-05T00:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-05T00:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-05T00:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T00:41:51,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-12-05T00:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6680 sec 2024-12-05T00:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,521 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/60037a2ab2534d12ba5696b3d047f310 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/60037a2ab2534d12ba5696b3d047f310 2024-12-05T00:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,529 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into 60037a2ab2534d12ba5696b3d047f310(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:51,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:51,529 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359311065; duration=0sec 2024-12-05T00:41:51,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:51,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:51,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:41:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:51,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:51,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:51,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:51,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:51,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:51,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052610f06964354dc68cfab4c1843d5195_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741997_1173 (size=27248) 2024-12-05T00:41:51,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:51,715 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/23f10e0b09334629b10c31474bcfe1ad as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/23f10e0b09334629b10c31474bcfe1ad 2024-12-05T00:41:51,722 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052610f06964354dc68cfab4c1843d5195_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052610f06964354dc68cfab4c1843d5195_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:51,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/cd268e97407640ddb3ef9e6a5a093dc4, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:51,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/cd268e97407640ddb3ef9e6a5a093dc4 is 175, key is test_row_0/A:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741996_1172 (size=83035) 2024-12-05T00:41:51,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/cd268e97407640ddb3ef9e6a5a093dc4 2024-12-05T00:41:51,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/0ba8c38aab56466088f7011f2de06fcf is 50, key is test_row_0/B:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:51,758 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 23f10e0b09334629b10c31474bcfe1ad(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:51,758 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:51,758 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359311066; duration=0sec 2024-12-05T00:41:51,759 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:51,759 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:41:51,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359371790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359371792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359371792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741998_1174 (size=12151) 2024-12-05T00:41:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T00:41:51,881 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:51,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-05T00:41:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:51,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:51,884 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:51,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359371901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359371902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359371902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359371961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:51,962 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:52,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T00:41:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359372112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359372112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359372112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T00:41:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/0ba8c38aab56466088f7011f2de06fcf 2024-12-05T00:41:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:52,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/2afce1ae657e4150b21914259facf066 is 50, key is test_row_0/C:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:52,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741999_1175 (size=12151) 2024-12-05T00:41:52,297 INFO [master/fea72ea5c4b6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T00:41:52,297 INFO [master/fea72ea5c4b6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T00:41:52,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T00:41:52,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:52,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359372421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359372422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359372422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T00:41:52,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:52,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:52,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/2afce1ae657e4150b21914259facf066 2024-12-05T00:41:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/cd268e97407640ddb3ef9e6a5a093dc4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4 2024-12-05T00:41:52,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4, entries=450, sequenceid=193, filesize=81.1 K 2024-12-05T00:41:52,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/0ba8c38aab56466088f7011f2de06fcf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0ba8c38aab56466088f7011f2de06fcf 2024-12-05T00:41:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0ba8c38aab56466088f7011f2de06fcf, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T00:41:52,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/2afce1ae657e4150b21914259facf066 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/2afce1ae657e4150b21914259facf066 2024-12-05T00:41:52,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/2afce1ae657e4150b21914259facf066, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T00:41:52,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c39421bab8284c2f34cba4e4ea472def in 968ms, sequenceid=193, compaction requested=true 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:52,638 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:52,638 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:52,639 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:52,639 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 145587 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:52,639 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:41:52,639 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:41:52,639 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,639 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,639 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/60037a2ab2534d12ba5696b3d047f310, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/9728828e37c34aabb60da22da7b5cd3d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0ba8c38aab56466088f7011f2de06fcf] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=35.9 K 2024-12-05T00:41:52,639 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/10324e5db35844f7b6af16aa6107b1ff, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=142.2 K 2024-12-05T00:41:52,639 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,639 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/10324e5db35844f7b6af16aa6107b1ff, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4] 2024-12-05T00:41:52,640 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 60037a2ab2534d12ba5696b3d047f310, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359308711 2024-12-05T00:41:52,640 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10324e5db35844f7b6af16aa6107b1ff, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359308711 2024-12-05T00:41:52,640 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b427986c5d604defb2c49dd864aa19fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733359309780 2024-12-05T00:41:52,640 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9728828e37c34aabb60da22da7b5cd3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733359309780 2024-12-05T00:41:52,640 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba8c38aab56466088f7011f2de06fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359311662 2024-12-05T00:41:52,641 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd268e97407640ddb3ef9e6a5a093dc4, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359311586 2024-12-05T00:41:52,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:52,649 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:52,650 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:52,650 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/5e43a2cb519d4dd79bd65072b9a3701c is 50, key is test_row_0/B:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:52,653 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c106d41fe2044c708191f1f247bf8ad4_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:52,656 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c106d41fe2044c708191f1f247bf8ad4_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:52,657 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c106d41fe2044c708191f1f247bf8ad4_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:52,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058dd1f8049feb4193afc849d01107c461_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359311733/Put/seqid=0 2024-12-05T00:41:52,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742000_1176 (size=12595) 2024-12-05T00:41:52,679 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/5e43a2cb519d4dd79bd65072b9a3701c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/5e43a2cb519d4dd79bd65072b9a3701c 2024-12-05T00:41:52,686 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into 5e43a2cb519d4dd79bd65072b9a3701c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:52,686 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:52,686 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359312638; duration=0sec 2024-12-05T00:41:52,686 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:52,686 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:41:52,686 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:52,688 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:52,688 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:41:52,688 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:52,688 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/23f10e0b09334629b10c31474bcfe1ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/2afce1ae657e4150b21914259facf066] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=35.9 K 2024-12-05T00:41:52,688 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f10e0b09334629b10c31474bcfe1ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359308711 2024-12-05T00:41:52,689 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ce295c3bcc4eb1bac0f5220bb4eb0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733359309780 2024-12-05T00:41:52,689 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2afce1ae657e4150b21914259facf066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359311662 2024-12-05T00:41:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742001_1177 (size=4469) 2024-12-05T00:41:52,700 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:52,701 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/414b15c215fc41b08520f3963edc326a is 50, key is test_row_0/C:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:52,702 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#148 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:52,703 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/34ac52c486e14c0fbf7fb0e5db11078b is 175, key is test_row_0/A:col10/1733359311668/Put/seqid=0 2024-12-05T00:41:52,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742002_1178 (size=12304) 2024-12-05T00:41:52,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742003_1179 (size=12595) 2024-12-05T00:41:52,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742004_1180 (size=31549) 2024-12-05T00:41:52,732 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/34ac52c486e14c0fbf7fb0e5db11078b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/34ac52c486e14c0fbf7fb0e5db11078b 2024-12-05T00:41:52,742 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into 34ac52c486e14c0fbf7fb0e5db11078b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:52,742 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:52,742 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359312638; duration=0sec 2024-12-05T00:41:52,742 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:52,742 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:41:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:52,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359372927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359372932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359372932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359372932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:53,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359373042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:53,110 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058dd1f8049feb4193afc849d01107c461_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058dd1f8049feb4193afc849d01107c461_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/62623e15bd164cc79509883a96e43d9f, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/62623e15bd164cc79509883a96e43d9f is 175, key is test_row_0/A:col10/1733359311733/Put/seqid=0 2024-12-05T00:41:53,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742005_1181 (size=31105) 2024-12-05T00:41:53,124 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/414b15c215fc41b08520f3963edc326a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/414b15c215fc41b08520f3963edc326a 2024-12-05T00:41:53,131 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 414b15c215fc41b08520f3963edc326a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:53,131 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:53,131 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359312638; duration=0sec 2024-12-05T00:41:53,131 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:53,131 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:41:53,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359373251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/62623e15bd164cc79509883a96e43d9f 2024-12-05T00:41:53,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/4376df782b794bf394aa8609cfab8689 is 50, key is test_row_0/B:col10/1733359311733/Put/seqid=0 2024-12-05T00:41:53,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742006_1182 (size=12151) 2024-12-05T00:41:53,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359373562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/4376df782b794bf394aa8609cfab8689 2024-12-05T00:41:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359373941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359373942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,943 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:53,943 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/cbb55aff6e734d2b8da83ff13747fadd is 50, key is test_row_0/C:col10/1733359311733/Put/seqid=0 2024-12-05T00:41:53,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742007_1183 (size=12151) 2024-12-05T00:41:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359373952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,953 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:53,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359373971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:53,975 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359374071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:54,351 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/cbb55aff6e734d2b8da83ff13747fadd 2024-12-05T00:41:54,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/62623e15bd164cc79509883a96e43d9f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f 2024-12-05T00:41:54,359 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f, entries=150, sequenceid=214, filesize=30.4 K 2024-12-05T00:41:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/4376df782b794bf394aa8609cfab8689 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4376df782b794bf394aa8609cfab8689 2024-12-05T00:41:54,364 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4376df782b794bf394aa8609cfab8689, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T00:41:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/cbb55aff6e734d2b8da83ff13747fadd as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cbb55aff6e734d2b8da83ff13747fadd 2024-12-05T00:41:54,370 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cbb55aff6e734d2b8da83ff13747fadd, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T00:41:54,370 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c39421bab8284c2f34cba4e4ea472def in 1722ms, sequenceid=214, compaction requested=false 2024-12-05T00:41:54,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:54,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:54,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-05T00:41:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-05T00:41:54,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T00:41:54,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-12-05T00:41:54,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4910 sec 2024-12-05T00:41:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:55,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:55,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b1c36f43596f46c2867289283c1ef246_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742008_1184 (size=12304) 2024-12-05T00:41:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359375137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359375241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359375451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:55,494 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:55,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b1c36f43596f46c2867289283c1ef246_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b1c36f43596f46c2867289283c1ef246_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:55,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/95fb2dd9b670486594e0927228731c2a, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:55,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/95fb2dd9b670486594e0927228731c2a is 175, key is test_row_0/A:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:55,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742009_1185 (size=31105) 2024-12-05T00:41:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359375761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:55,904 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/95fb2dd9b670486594e0927228731c2a 2024-12-05T00:41:55,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/fce0ae215de443e19258fb908dc3fbc6 is 50, key is test_row_0/B:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:55,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742010_1186 (size=12151) 2024-12-05T00:41:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359375962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359375962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,966 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:55,966 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:55,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359375981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:55,984 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T00:41:56,052 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:56,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-05T00:41:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T00:41:56,054 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:56,055 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:56,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T00:41:56,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T00:41:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359376272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:56,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/fce0ae215de443e19258fb908dc3fbc6 2024-12-05T00:41:56,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/6c43712e27754979a7ad6f20e7d59ffa is 50, key is test_row_0/C:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:56,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742011_1187 (size=12151) 2024-12-05T00:41:56,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T00:41:56,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T00:41:56,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T00:41:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T00:41:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T00:41:56,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/6c43712e27754979a7ad6f20e7d59ffa 2024-12-05T00:41:56,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/95fb2dd9b670486594e0927228731c2a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a 2024-12-05T00:41:56,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a, entries=150, sequenceid=230, filesize=30.4 K 2024-12-05T00:41:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/fce0ae215de443e19258fb908dc3fbc6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/fce0ae215de443e19258fb908dc3fbc6 2024-12-05T00:41:56,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/fce0ae215de443e19258fb908dc3fbc6, entries=150, sequenceid=230, filesize=11.9 K 2024-12-05T00:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/6c43712e27754979a7ad6f20e7d59ffa as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6c43712e27754979a7ad6f20e7d59ffa 2024-12-05T00:41:56,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6c43712e27754979a7ad6f20e7d59ffa, entries=150, sequenceid=230, filesize=11.9 K 2024-12-05T00:41:56,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c39421bab8284c2f34cba4e4ea472def in 1662ms, sequenceid=230, compaction requested=true 2024-12-05T00:41:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:56,745 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:56,746 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:56,747 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:56,747 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:41:56,747 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,747 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/34ac52c486e14c0fbf7fb0e5db11078b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=91.6 K 2024-12-05T00:41:56,747 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,747 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/34ac52c486e14c0fbf7fb0e5db11078b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a] 2024-12-05T00:41:56,748 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:56,748 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ac52c486e14c0fbf7fb0e5db11078b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359311662 2024-12-05T00:41:56,748 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:41:56,748 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,748 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/5e43a2cb519d4dd79bd65072b9a3701c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4376df782b794bf394aa8609cfab8689, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/fce0ae215de443e19258fb908dc3fbc6] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.0 K 2024-12-05T00:41:56,748 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 62623e15bd164cc79509883a96e43d9f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733359311708 2024-12-05T00:41:56,748 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e43a2cb519d4dd79bd65072b9a3701c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359311662 2024-12-05T00:41:56,748 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 95fb2dd9b670486594e0927228731c2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733359312921 2024-12-05T00:41:56,749 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4376df782b794bf394aa8609cfab8689, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733359311708 2024-12-05T00:41:56,749 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce0ae215de443e19258fb908dc3fbc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733359312921 2024-12-05T00:41:56,755 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:56,757 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:56,758 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/46254b1145cc4d599cf336189b764613 is 50, key is test_row_0/B:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:56,758 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205df6c46223c6f44568f0b4494e32f2975_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:56,760 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205df6c46223c6f44568f0b4494e32f2975_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:56,760 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205df6c46223c6f44568f0b4494e32f2975_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:56,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742012_1188 (size=12697) 2024-12-05T00:41:56,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742013_1189 (size=4469) 2024-12-05T00:41:56,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:56,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059d0ae782f7684f429a91a2b7e0cebaa5_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359315125/Put/seqid=0 2024-12-05T00:41:56,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742014_1190 (size=12304) 2024-12-05T00:41:57,170 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:57,170 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/88e9a70fadfd4fb8bb8b350b04a9f9fa is 175, key is test_row_0/A:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:57,171 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/46254b1145cc4d599cf336189b764613 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/46254b1145cc4d599cf336189b764613 2024-12-05T00:41:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742015_1191 (size=31651) 2024-12-05T00:41:57,177 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into 46254b1145cc4d599cf336189b764613(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:57,177 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:57,177 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359316746; duration=0sec 2024-12-05T00:41:57,178 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:57,178 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:41:57,178 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:57,179 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:57,180 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:41:57,180 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:57,180 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/414b15c215fc41b08520f3963edc326a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cbb55aff6e734d2b8da83ff13747fadd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6c43712e27754979a7ad6f20e7d59ffa] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.0 K 2024-12-05T00:41:57,180 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 414b15c215fc41b08520f3963edc326a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733359311662 2024-12-05T00:41:57,181 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/88e9a70fadfd4fb8bb8b350b04a9f9fa as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/88e9a70fadfd4fb8bb8b350b04a9f9fa 2024-12-05T00:41:57,181 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb55aff6e734d2b8da83ff13747fadd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733359311708 2024-12-05T00:41:57,181 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c43712e27754979a7ad6f20e7d59ffa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733359312921 2024-12-05T00:41:57,185 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into 88e9a70fadfd4fb8bb8b350b04a9f9fa(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:57,185 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:57,186 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359316745; duration=0sec 2024-12-05T00:41:57,186 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:57,186 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:41:57,189 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:57,189 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/68d1cfe2a13244c6aeca39474a148b50 is 50, key is test_row_0/C:col10/1733359312926/Put/seqid=0 2024-12-05T00:41:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T00:41:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742016_1192 (size=12697) 2024-12-05T00:41:57,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:57,232 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059d0ae782f7684f429a91a2b7e0cebaa5_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059d0ae782f7684f429a91a2b7e0cebaa5_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/927f08050ad64a79b53a6af41a07f708, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/927f08050ad64a79b53a6af41a07f708 is 175, key is test_row_0/A:col10/1733359315125/Put/seqid=0 2024-12-05T00:41:57,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742017_1193 (size=31105) 2024-12-05T00:41:57,237 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/927f08050ad64a79b53a6af41a07f708 2024-12-05T00:41:57,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/06ccce9dbec14e1aa39a6d2b5a55c6a7 is 50, key is test_row_0/B:col10/1733359315125/Put/seqid=0 2024-12-05T00:41:57,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742018_1194 (size=12151) 2024-12-05T00:41:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:57,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359377314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:57,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359377421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:57,601 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/68d1cfe2a13244c6aeca39474a148b50 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/68d1cfe2a13244c6aeca39474a148b50 2024-12-05T00:41:57,606 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 68d1cfe2a13244c6aeca39474a148b50(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:57,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:57,606 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359316746; duration=0sec 2024-12-05T00:41:57,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:57,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:41:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359377631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:57,649 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/06ccce9dbec14e1aa39a6d2b5a55c6a7 2024-12-05T00:41:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/24f6b59c40a54de6aca5afb91fa46592 is 50, key is test_row_0/C:col10/1733359315125/Put/seqid=0 2024-12-05T00:41:57,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742019_1195 (size=12151) 2024-12-05T00:41:57,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/24f6b59c40a54de6aca5afb91fa46592 2024-12-05T00:41:57,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/927f08050ad64a79b53a6af41a07f708 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708 2024-12-05T00:41:57,669 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708, entries=150, sequenceid=249, filesize=30.4 K 2024-12-05T00:41:57,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/06ccce9dbec14e1aa39a6d2b5a55c6a7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/06ccce9dbec14e1aa39a6d2b5a55c6a7 2024-12-05T00:41:57,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/06ccce9dbec14e1aa39a6d2b5a55c6a7, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T00:41:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/24f6b59c40a54de6aca5afb91fa46592 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/24f6b59c40a54de6aca5afb91fa46592 2024-12-05T00:41:57,681 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/24f6b59c40a54de6aca5afb91fa46592, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T00:41:57,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c39421bab8284c2f34cba4e4ea472def in 864ms, sequenceid=249, compaction requested=false 2024-12-05T00:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-05T00:41:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-05T00:41:57,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-05T00:41:57,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-12-05T00:41:57,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6340 sec 2024-12-05T00:41:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:57,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T00:41:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:57,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056888e933ade141d3b44f4f783b100d00_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:57,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742020_1196 (size=14994) 2024-12-05T00:41:58,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359378004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1733359378011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:58,014 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:41:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:58,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359378112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T00:41:58,202 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:41:58,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:41:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-05T00:41:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T00:41:58,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:41:58,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:41:58,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:41:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T00:41:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359378322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:58,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:58,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:58,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,370 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:58,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056888e933ade141d3b44f4f783b100d00_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056888e933ade141d3b44f4f783b100d00_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:58,375 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ee49c879fcec43a59c036afa5d3d400b, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:58,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ee49c879fcec43a59c036afa5d3d400b is 175, key is test_row_0/A:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:58,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742021_1197 (size=39949) 2024-12-05T00:41:58,379 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ee49c879fcec43a59c036afa5d3d400b 2024-12-05T00:41:58,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/50556429478843e7858f89ffaf427a1e is 50, key is test_row_0/B:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:58,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742022_1198 (size=12301) 2024-12-05T00:41:58,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:58,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:58,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T00:41:58,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359378631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:58,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:58,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/50556429478843e7858f89ffaf427a1e 2024-12-05T00:41:58,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/cad7c1025fe145699e6c70cb2ecd1545 is 50, key is test_row_0/C:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:58,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742023_1199 (size=12301) 2024-12-05T00:41:58,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T00:41:58,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:58,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:59,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:41:59,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:41:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359379141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:41:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:41:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:41:59,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/cad7c1025fe145699e6c70cb2ecd1545 2024-12-05T00:41:59,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ee49c879fcec43a59c036afa5d3d400b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b 2024-12-05T00:41:59,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b, entries=200, sequenceid=266, filesize=39.0 K 2024-12-05T00:41:59,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/50556429478843e7858f89ffaf427a1e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/50556429478843e7858f89ffaf427a1e 2024-12-05T00:41:59,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/50556429478843e7858f89ffaf427a1e, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T00:41:59,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/cad7c1025fe145699e6c70cb2ecd1545 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cad7c1025fe145699e6c70cb2ecd1545 2024-12-05T00:41:59,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cad7c1025fe145699e6c70cb2ecd1545, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T00:41:59,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c39421bab8284c2f34cba4e4ea472def in 1279ms, sequenceid=266, compaction requested=true 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:41:59,226 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:59,226 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:41:59,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:59,227 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:59,227 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:41:59,227 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,227 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/46254b1145cc4d599cf336189b764613, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/06ccce9dbec14e1aa39a6d2b5a55c6a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/50556429478843e7858f89ffaf427a1e] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.3 K 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 46254b1145cc4d599cf336189b764613, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733359312921 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ccce9dbec14e1aa39a6d2b5a55c6a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733359315125 2024-12-05T00:41:59,228 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,228 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/88e9a70fadfd4fb8bb8b350b04a9f9fa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=100.3 K 2024-12-05T00:41:59,228 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/88e9a70fadfd4fb8bb8b350b04a9f9fa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b] 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 50556429478843e7858f89ffaf427a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733359317297 2024-12-05T00:41:59,228 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e9a70fadfd4fb8bb8b350b04a9f9fa, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733359312921 2024-12-05T00:41:59,229 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 927f08050ad64a79b53a6af41a07f708, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733359315125 2024-12-05T00:41:59,229 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee49c879fcec43a59c036afa5d3d400b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733359317297 2024-12-05T00:41:59,234 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#165 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:59,235 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/d95debdb10394c92a70462c8914b5ec8 is 50, key is test_row_0/B:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:59,236 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:59,238 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052275e21f54f94a8bb41ed60dad969a23_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:59,239 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052275e21f54f94a8bb41ed60dad969a23_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:59,239 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052275e21f54f94a8bb41ed60dad969a23_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:59,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742024_1200 (size=12949) 2024-12-05T00:41:59,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742025_1201 (size=4469) 2024-12-05T00:41:59,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-05T00:41:59,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,273 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T00:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:41:59,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120546592847324a4cd2af6cf3fa1ebd7438_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359317999/Put/seqid=0 2024-12-05T00:41:59,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742026_1202 (size=12454) 2024-12-05T00:41:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:41:59,291 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120546592847324a4cd2af6cf3fa1ebd7438_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120546592847324a4cd2af6cf3fa1ebd7438_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:59,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/4d146e755be744cca0a19a31967ec87c, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:41:59,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/4d146e755be744cca0a19a31967ec87c is 175, key is test_row_0/A:col10/1733359317999/Put/seqid=0 2024-12-05T00:41:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742027_1203 (size=31255) 2024-12-05T00:41:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T00:41:59,647 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:59,648 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ef13f6e59f0b4444a7298edc447188c7 is 175, key is test_row_0/A:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:59,650 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/d95debdb10394c92a70462c8914b5ec8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/d95debdb10394c92a70462c8914b5ec8 2024-12-05T00:41:59,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742028_1204 (size=31903) 2024-12-05T00:41:59,655 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into d95debdb10394c92a70462c8914b5ec8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:59,655 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:59,655 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359319226; duration=0sec 2024-12-05T00:41:59,656 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:41:59,656 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:41:59,656 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:41:59,657 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ef13f6e59f0b4444a7298edc447188c7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ef13f6e59f0b4444a7298edc447188c7 2024-12-05T00:41:59,657 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:41:59,657 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:41:59,657 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:41:59,657 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/68d1cfe2a13244c6aeca39474a148b50, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/24f6b59c40a54de6aca5afb91fa46592, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cad7c1025fe145699e6c70cb2ecd1545] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.3 K 2024-12-05T00:41:59,657 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d1cfe2a13244c6aeca39474a148b50, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733359312921 2024-12-05T00:41:59,658 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f6b59c40a54de6aca5afb91fa46592, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733359315125 2024-12-05T00:41:59,658 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting cad7c1025fe145699e6c70cb2ecd1545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733359317297 2024-12-05T00:41:59,662 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into ef13f6e59f0b4444a7298edc447188c7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:41:59,662 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:41:59,662 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359319226; duration=0sec 2024-12-05T00:41:59,662 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:41:59,663 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:41:59,668 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:41:59,668 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/116994c52825402cb081db115be504be is 50, key is test_row_0/C:col10/1733359317944/Put/seqid=0 2024-12-05T00:41:59,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742029_1205 (size=12949) 2024-12-05T00:41:59,697 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/4d146e755be744cca0a19a31967ec87c 2024-12-05T00:41:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/c0225de1b5e64c24a5c4b94c0d4036a7 is 50, key is test_row_0/B:col10/1733359317999/Put/seqid=0 2024-12-05T00:41:59,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742030_1206 (size=12301) 2024-12-05T00:41:59,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/c0225de1b5e64c24a5c4b94c0d4036a7 2024-12-05T00:41:59,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/ef7bbf09ce4c4054aaa4f24a853541c7 is 50, key is test_row_0/C:col10/1733359317999/Put/seqid=0 2024-12-05T00:41:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742031_1207 (size=12301) 2024-12-05T00:41:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:41:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359380000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359380004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45054 deadline: 1733359380022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,025 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,088 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/116994c52825402cb081db115be504be as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/116994c52825402cb081db115be504be 2024-12-05T00:42:00,093 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 116994c52825402cb081db115be504be(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:00,093 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:00,093 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359319226; duration=0sec 2024-12-05T00:42:00,093 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:00,093 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:42:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359380111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359380112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,128 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/ef7bbf09ce4c4054aaa4f24a853541c7 2024-12-05T00:42:00,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/4d146e755be744cca0a19a31967ec87c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c 2024-12-05T00:42:00,135 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c, entries=150, sequenceid=284, filesize=30.5 K 2024-12-05T00:42:00,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/c0225de1b5e64c24a5c4b94c0d4036a7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c0225de1b5e64c24a5c4b94c0d4036a7 2024-12-05T00:42:00,140 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c0225de1b5e64c24a5c4b94c0d4036a7, entries=150, sequenceid=284, filesize=12.0 K 2024-12-05T00:42:00,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/ef7bbf09ce4c4054aaa4f24a853541c7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ef7bbf09ce4c4054aaa4f24a853541c7 2024-12-05T00:42:00,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ef7bbf09ce4c4054aaa4f24a853541c7, entries=150, sequenceid=284, filesize=12.0 K 2024-12-05T00:42:00,146 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c39421bab8284c2f34cba4e4ea472def in 872ms, sequenceid=284, compaction requested=false 2024-12-05T00:42:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-05T00:42:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-05T00:42:00,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-05T00:42:00,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9420 sec 2024-12-05T00:42:00,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9470 sec 2024-12-05T00:42:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:00,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T00:42:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:42:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:42:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:42:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:00,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050f49bccb770842f3997bbe102ad4c625_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:00,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742032_1208 (size=14994) 2024-12-05T00:42:00,185 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:00,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050f49bccb770842f3997bbe102ad4c625_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050f49bccb770842f3997bbe102ad4c625_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:00,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/73ce3175be6648818260fac40e24b27c, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:00,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/73ce3175be6648818260fac40e24b27c is 175, key is test_row_0/A:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:00,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742033_1209 (size=39949) 2024-12-05T00:42:00,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359380221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359380321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359380321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359380332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T00:42:00,352 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:00,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-05T00:42:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:00,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:00,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:00,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:00,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:00,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359380542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/73ce3175be6648818260fac40e24b27c 2024-12-05T00:42:00,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/110f1662b3b0470aad099c5a566130f5 is 50, key is test_row_0/B:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:00,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742034_1210 (size=12301) 2024-12-05T00:42:00,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359380631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359380631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:00,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359380851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:00,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:00,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:01,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/110f1662b3b0470aad099c5a566130f5 2024-12-05T00:42:01,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/c2a3649198e74473a011661ba88ee060 is 50, key is test_row_0/C:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:01,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742035_1211 (size=12301) 2024-12-05T00:42:01,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359381142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359381142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:01,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359381362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:01,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:01,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/c2a3649198e74473a011661ba88ee060 2024-12-05T00:42:01,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/73ce3175be6648818260fac40e24b27c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c 2024-12-05T00:42:01,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c, entries=200, sequenceid=302, filesize=39.0 K 2024-12-05T00:42:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/110f1662b3b0470aad099c5a566130f5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/110f1662b3b0470aad099c5a566130f5 2024-12-05T00:42:01,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/110f1662b3b0470aad099c5a566130f5, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T00:42:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/c2a3649198e74473a011661ba88ee060 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/c2a3649198e74473a011661ba88ee060 2024-12-05T00:42:01,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/c2a3649198e74473a011661ba88ee060, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T00:42:01,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c39421bab8284c2f34cba4e4ea472def in 1274ms, sequenceid=302, compaction requested=true 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:01,445 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:01,445 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:01,446 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:01,446 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:42:01,446 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,446 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ef13f6e59f0b4444a7298edc447188c7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=100.7 K 2024-12-05T00:42:01,446 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,447 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ef13f6e59f0b4444a7298edc447188c7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c] 2024-12-05T00:42:01,447 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:01,447 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:42:01,447 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,447 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/d95debdb10394c92a70462c8914b5ec8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c0225de1b5e64c24a5c4b94c0d4036a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/110f1662b3b0470aad099c5a566130f5] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.7 K 2024-12-05T00:42:01,449 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef13f6e59f0b4444a7298edc447188c7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733359317297 2024-12-05T00:42:01,449 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d95debdb10394c92a70462c8914b5ec8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733359317297 2024-12-05T00:42:01,449 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d146e755be744cca0a19a31967ec87c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733359317981 2024-12-05T00:42:01,449 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c0225de1b5e64c24a5c4b94c0d4036a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733359317981 2024-12-05T00:42:01,450 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ce3175be6648818260fac40e24b27c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733359319997 2024-12-05T00:42:01,450 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 110f1662b3b0470aad099c5a566130f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733359319997 2024-12-05T00:42:01,464 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:01,464 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/4ac2b4b2e7b84bd5a94ca10acb56a563 is 50, key is test_row_0/B:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:01,466 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:01,474 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412057f90255de83a4dde81f00922f21de3fb_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:01,476 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412057f90255de83a4dde81f00922f21de3fb_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:01,476 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057f90255de83a4dde81f00922f21de3fb_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742036_1212 (size=13051) 2024-12-05T00:42:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:01,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742037_1213 (size=4469) 2024-12-05T00:42:01,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,576 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120545563c243b0b4a1f862e87176a5d2ecc_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359320220/Put/seqid=0 2024-12-05T00:42:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742038_1214 (size=12454) 2024-12-05T00:42:01,884 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/4ac2b4b2e7b84bd5a94ca10acb56a563 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4ac2b4b2e7b84bd5a94ca10acb56a563 2024-12-05T00:42:01,897 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into 4ac2b4b2e7b84bd5a94ca10acb56a563(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:01,897 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:01,897 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359321445; duration=0sec 2024-12-05T00:42:01,897 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:01,897 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:42:01,897 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:01,898 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:01,899 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:42:01,899 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:01,899 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/116994c52825402cb081db115be504be, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ef7bbf09ce4c4054aaa4f24a853541c7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/c2a3649198e74473a011661ba88ee060] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.7 K 2024-12-05T00:42:01,899 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 116994c52825402cb081db115be504be, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733359317297 2024-12-05T00:42:01,900 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7bbf09ce4c4054aaa4f24a853541c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733359317981 2024-12-05T00:42:01,900 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a3649198e74473a011661ba88ee060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733359319997 2024-12-05T00:42:01,904 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:01,905 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/7749071694814e8ab642e0563bf57a9b is 175, key is test_row_0/A:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:01,930 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:01,931 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/f6076d7f4e1b464eac7e04010e1624ad is 50, key is test_row_0/C:col10/1733359320162/Put/seqid=0 2024-12-05T00:42:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742039_1215 (size=32005) 2024-12-05T00:42:01,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742040_1216 (size=13051) 2024-12-05T00:42:01,965 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/f6076d7f4e1b464eac7e04010e1624ad as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f6076d7f4e1b464eac7e04010e1624ad 2024-12-05T00:42:01,973 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into f6076d7f4e1b464eac7e04010e1624ad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:01,973 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:01,973 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359321445; duration=0sec 2024-12-05T00:42:01,974 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:01,974 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:42:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:02,011 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120545563c243b0b4a1f862e87176a5d2ecc_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120545563c243b0b4a1f862e87176a5d2ecc_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ec0a90f5a9134339a825cc8cb21d0c02, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ec0a90f5a9134339a825cc8cb21d0c02 is 175, key is test_row_0/A:col10/1733359320220/Put/seqid=0 2024-12-05T00:42:02,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742041_1217 (size=31255) 2024-12-05T00:42:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. as already flushing 2024-12-05T00:42:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359382179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359382183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,267 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-12-05T00:42:02,267 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:02,267 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,267 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:02,268 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-12-05T00:42:02,268 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:02,268 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,268 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:02,271 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-12-05T00:42:02,271 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:02,271 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,271 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:02,274 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-12-05T00:42:02,274 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:02,274 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,274 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:02,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:02,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359382292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359382292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,351 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/7749071694814e8ab642e0563bf57a9b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/7749071694814e8ab642e0563bf57a9b 2024-12-05T00:42:02,356 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into 7749071694814e8ab642e0563bf57a9b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:02,356 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:02,356 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359321445; duration=0sec 2024-12-05T00:42:02,357 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:02,357 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:42:02,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45034 deadline: 1733359382381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,382 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in c39421bab8284c2f34cba4e4ea472def of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,422 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ec0a90f5a9134339a825cc8cb21d0c02 2024-12-05T00:42:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/98a134510f864d058261caa87d2a7dca is 50, key is test_row_0/B:col10/1733359320220/Put/seqid=0 2024-12-05T00:42:02,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742042_1218 (size=12301) 2024-12-05T00:42:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:02,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359382502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359382503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45068 deadline: 1733359382812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45012 deadline: 1733359382813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:02,838 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/98a134510f864d058261caa87d2a7dca 2024-12-05T00:42:02,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/ffd74819511d4f09aa0c867792d90780 is 50, key is test_row_0/C:col10/1733359320220/Put/seqid=0 2024-12-05T00:42:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742043_1219 (size=12301) 2024-12-05T00:42:03,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/ffd74819511d4f09aa0c867792d90780 2024-12-05T00:42:03,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/ec0a90f5a9134339a825cc8cb21d0c02 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02 2024-12-05T00:42:03,271 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02, entries=150, sequenceid=319, filesize=30.5 K 2024-12-05T00:42:03,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/98a134510f864d058261caa87d2a7dca as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/98a134510f864d058261caa87d2a7dca 2024-12-05T00:42:03,277 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/98a134510f864d058261caa87d2a7dca, entries=150, sequenceid=319, filesize=12.0 K 2024-12-05T00:42:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/ffd74819511d4f09aa0c867792d90780 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ffd74819511d4f09aa0c867792d90780 2024-12-05T00:42:03,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ffd74819511d4f09aa0c867792d90780, entries=150, sequenceid=319, filesize=12.0 K 2024-12-05T00:42:03,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c39421bab8284c2f34cba4e4ea472def in 1706ms, sequenceid=319, compaction requested=false 2024-12-05T00:42:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-05T00:42:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-05T00:42:03,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-05T00:42:03,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9290 sec 2024-12-05T00:42:03,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.9330 sec 2024-12-05T00:42:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:03,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T00:42:03,323 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-12-05T00:42:03,323 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-12-05T00:42:03,323 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:03,323 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:03,323 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:03,323 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:03,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:42:03,324 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:03,324 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:03,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:03,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:42:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:42:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:03,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cd91bbcfaeb14b30be59feeaa0c3d4b3_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:03,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742044_1220 (size=12454) 2024-12-05T00:42:03,739 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:03,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cd91bbcfaeb14b30be59feeaa0c3d4b3_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cd91bbcfaeb14b30be59feeaa0c3d4b3_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:03,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/bd344d878312405b969ddb80c05d88a8, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:03,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/bd344d878312405b969ddb80c05d88a8 is 175, key is test_row_0/A:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:03,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742045_1221 (size=31255) 2024-12-05T00:42:04,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/bd344d878312405b969ddb80c05d88a8 2024-12-05T00:42:04,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/1f0487840a3e4f7db4d2baa3d2f5bd48 is 50, key is test_row_0/B:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:04,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742046_1222 (size=12301) 2024-12-05T00:42:04,406 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-12-05T00:42:04,406 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:04,406 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:04,407 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:04,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T00:42:04,523 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:04,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/1f0487840a3e4f7db4d2baa3d2f5bd48 2024-12-05T00:42:04,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/054f3fd8df324b19b516698a4f06f2cd is 50, key is test_row_0/C:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:04,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742047_1223 (size=12301) 2024-12-05T00:42:04,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/054f3fd8df324b19b516698a4f06f2cd 2024-12-05T00:42:05,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/bd344d878312405b969ddb80c05d88a8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8 2024-12-05T00:42:05,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8, entries=150, sequenceid=339, filesize=30.5 K 2024-12-05T00:42:05,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/1f0487840a3e4f7db4d2baa3d2f5bd48 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/1f0487840a3e4f7db4d2baa3d2f5bd48 2024-12-05T00:42:05,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/1f0487840a3e4f7db4d2baa3d2f5bd48, entries=150, sequenceid=339, filesize=12.0 K 2024-12-05T00:42:05,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/054f3fd8df324b19b516698a4f06f2cd as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/054f3fd8df324b19b516698a4f06f2cd 2024-12-05T00:42:05,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/054f3fd8df324b19b516698a4f06f2cd, entries=150, sequenceid=339, filesize=12.0 K 2024-12-05T00:42:05,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for c39421bab8284c2f34cba4e4ea472def in 1696ms, sequenceid=339, compaction requested=true 2024-12-05T00:42:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:05,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:05,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:05,020 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:05,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c39421bab8284c2f34cba4e4ea472def:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:05,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:05,020 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/B is initiating minor compaction (all files) 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/A is initiating minor compaction (all files) 2024-12-05T00:42:05,021 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/A in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:05,021 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/B in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:05,021 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/7749071694814e8ab642e0563bf57a9b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=92.3 K 2024-12-05T00:42:05,021 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4ac2b4b2e7b84bd5a94ca10acb56a563, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/98a134510f864d058261caa87d2a7dca, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/1f0487840a3e4f7db4d2baa3d2f5bd48] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.8 K 2024-12-05T00:42:05,021 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/7749071694814e8ab642e0563bf57a9b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8] 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac2b4b2e7b84bd5a94ca10acb56a563, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733359319997 2024-12-05T00:42:05,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7749071694814e8ab642e0563bf57a9b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733359319997 2024-12-05T00:42:05,022 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a134510f864d058261caa87d2a7dca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733359320203 2024-12-05T00:42:05,022 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0a90f5a9134339a825cc8cb21d0c02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733359320203 2024-12-05T00:42:05,022 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0487840a3e4f7db4d2baa3d2f5bd48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733359322181 2024-12-05T00:42:05,022 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd344d878312405b969ddb80c05d88a8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733359322181 2024-12-05T00:42:05,030 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:05,030 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/a39bcda9f60941788e4a356436293250 is 50, key is test_row_0/B:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:05,033 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742048_1224 (size=13153) 2024-12-05T00:42:05,035 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205ac90f12200694ed1a08485c8d9d192a3_c39421bab8284c2f34cba4e4ea472def store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:05,061 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205ac90f12200694ed1a08485c8d9d192a3_c39421bab8284c2f34cba4e4ea472def, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:05,062 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ac90f12200694ed1a08485c8d9d192a3_c39421bab8284c2f34cba4e4ea472def because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:05,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742049_1225 (size=4469) 2024-12-05T00:42:05,443 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/a39bcda9f60941788e4a356436293250 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a39bcda9f60941788e4a356436293250 2024-12-05T00:42:05,451 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/B of c39421bab8284c2f34cba4e4ea472def into a39bcda9f60941788e4a356436293250(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:05,451 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:05,451 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/B, priority=13, startTime=1733359325019; duration=0sec 2024-12-05T00:42:05,452 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:05,452 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:B 2024-12-05T00:42:05,452 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:05,453 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:05,453 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): c39421bab8284c2f34cba4e4ea472def/C is initiating minor compaction (all files) 2024-12-05T00:42:05,453 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c39421bab8284c2f34cba4e4ea472def/C in TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:05,453 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f6076d7f4e1b464eac7e04010e1624ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ffd74819511d4f09aa0c867792d90780, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/054f3fd8df324b19b516698a4f06f2cd] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp, totalSize=36.8 K 2024-12-05T00:42:05,454 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting f6076d7f4e1b464eac7e04010e1624ad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733359319997 2024-12-05T00:42:05,454 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ffd74819511d4f09aa0c867792d90780, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733359320203 2024-12-05T00:42:05,454 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 054f3fd8df324b19b516698a4f06f2cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733359322181 2024-12-05T00:42:05,464 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:05,465 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/55214cfe5a97472c91eb3a9e7789a27a is 50, key is test_row_0/C:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:05,467 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c39421bab8284c2f34cba4e4ea472def#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:05,467 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/d379d23d0b31491abe6fc4b9f86c2a03 is 175, key is test_row_0/A:col10/1733359323322/Put/seqid=0 2024-12-05T00:42:05,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742050_1226 (size=13153) 2024-12-05T00:42:05,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742051_1227 (size=32107) 2024-12-05T00:42:05,590 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T00:42:05,880 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/55214cfe5a97472c91eb3a9e7789a27a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/55214cfe5a97472c91eb3a9e7789a27a 2024-12-05T00:42:05,881 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/d379d23d0b31491abe6fc4b9f86c2a03 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d379d23d0b31491abe6fc4b9f86c2a03 2024-12-05T00:42:05,886 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/A of c39421bab8284c2f34cba4e4ea472def into d379d23d0b31491abe6fc4b9f86c2a03(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:05,886 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:05,886 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/A, priority=13, startTime=1733359325019; duration=0sec 2024-12-05T00:42:05,886 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c39421bab8284c2f34cba4e4ea472def/C of c39421bab8284c2f34cba4e4ea472def into 55214cfe5a97472c91eb3a9e7789a27a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:05,886 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:05,886 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c39421bab8284c2f34cba4e4ea472def: 2024-12-05T00:42:05,886 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:A 2024-12-05T00:42:05,886 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., storeName=c39421bab8284c2f34cba4e4ea472def/C, priority=13, startTime=1733359325020; duration=0sec 2024-12-05T00:42:05,886 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:05,887 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c39421bab8284c2f34cba4e4ea472def:C 2024-12-05T00:42:08,058 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-12-05T00:42:08,058 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:08,058 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:08,059 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:08,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:10,067 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-12-05T00:42:10,067 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:10,067 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:10,067 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:10,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4327 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4245 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1860 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5579 rows 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1861 2024-12-05T00:42:10,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5582 rows 2024-12-05T00:42:10,068 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:42:10,068 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:42:10,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:10,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:10,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:10,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T00:42:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T00:42:10,074 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359330073"}]},"ts":"1733359330073"} 2024-12-05T00:42:10,076 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T00:42:10,076 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T00:42:10,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:42:10,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, UNASSIGN}] 2024-12-05T00:42:10,080 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, UNASSIGN 2024-12-05T00:42:10,080 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:10,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, UNASSIGN because future has completed 2024-12-05T00:42:10,082 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:42:10,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:42:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T00:42:10,235 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:10,235 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:42:10,235 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing c39421bab8284c2f34cba4e4ea472def, disabling compactions & flushes 2024-12-05T00:42:10,235 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:10,235 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:10,235 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. after waiting 0 ms 2024-12-05T00:42:10,235 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:10,235 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing c39421bab8284c2f34cba4e4ea472def 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-05T00:42:10,236 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=A 2024-12-05T00:42:10,236 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:10,236 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=B 2024-12-05T00:42:10,236 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:10,236 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c39421bab8284c2f34cba4e4ea472def, store=C 2024-12-05T00:42:10,236 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:10,247 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ebf3025096dc40e88a47deadd4ab404e_c39421bab8284c2f34cba4e4ea472def is 50, key is test_row_0/A:col10/1733359328053/Put/seqid=0 2024-12-05T00:42:10,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742052_1228 (size=9914) 2024-12-05T00:42:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T00:42:10,653 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:10,664 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ebf3025096dc40e88a47deadd4ab404e_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ebf3025096dc40e88a47deadd4ab404e_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:10,665 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/e30b8b3cb82640daac5f94dc4f8a0e29, store: [table=TestAcidGuarantees family=A region=c39421bab8284c2f34cba4e4ea472def] 2024-12-05T00:42:10,665 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/e30b8b3cb82640daac5f94dc4f8a0e29 is 175, key is test_row_0/A:col10/1733359328053/Put/seqid=0 2024-12-05T00:42:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742053_1229 (size=22561) 2024-12-05T00:42:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T00:42:11,071 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/e30b8b3cb82640daac5f94dc4f8a0e29 2024-12-05T00:42:11,084 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/ba9f7a5339b34345af9fbb68cec2e200 is 50, key is test_row_0/B:col10/1733359328053/Put/seqid=0 2024-12-05T00:42:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742054_1230 (size=9857) 2024-12-05T00:42:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T00:42:11,490 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/ba9f7a5339b34345af9fbb68cec2e200 2024-12-05T00:42:11,504 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/df6018b163f24b3f8867725ea46bb7ab is 50, key is test_row_0/C:col10/1733359328053/Put/seqid=0 2024-12-05T00:42:11,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742055_1231 (size=9857) 2024-12-05T00:42:11,908 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/df6018b163f24b3f8867725ea46bb7ab 2024-12-05T00:42:11,914 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/A/e30b8b3cb82640daac5f94dc4f8a0e29 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/e30b8b3cb82640daac5f94dc4f8a0e29 2024-12-05T00:42:11,919 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/e30b8b3cb82640daac5f94dc4f8a0e29, entries=100, sequenceid=348, filesize=22.0 K 2024-12-05T00:42:11,920 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/B/ba9f7a5339b34345af9fbb68cec2e200 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ba9f7a5339b34345af9fbb68cec2e200 2024-12-05T00:42:11,925 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ba9f7a5339b34345af9fbb68cec2e200, entries=100, sequenceid=348, filesize=9.6 K 2024-12-05T00:42:11,926 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/.tmp/C/df6018b163f24b3f8867725ea46bb7ab as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/df6018b163f24b3f8867725ea46bb7ab 2024-12-05T00:42:11,930 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/df6018b163f24b3f8867725ea46bb7ab, entries=100, sequenceid=348, filesize=9.6 K 2024-12-05T00:42:11,931 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c39421bab8284c2f34cba4e4ea472def in 1696ms, sequenceid=348, compaction requested=false 2024-12-05T00:42:11,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/525259769ab0475a8ee2e97e39c4a803, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d86cc77a7ca14ca9a9bb5ca68bb14372, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/10324e5db35844f7b6af16aa6107b1ff, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/34ac52c486e14c0fbf7fb0e5db11078b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/88e9a70fadfd4fb8bb8b350b04a9f9fa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ef13f6e59f0b4444a7298edc447188c7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/7749071694814e8ab642e0563bf57a9b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8] to archive 2024-12-05T00:42:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:42:11,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/492e62bb6d1a414fb6e9a6f61d2b8e7a 2024-12-05T00:42:11,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/72b6401806d74453b175405646df19ad 2024-12-05T00:42:11,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/31c8c107ab3643bc903a5f5d7af6946f 2024-12-05T00:42:11,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/525259769ab0475a8ee2e97e39c4a803 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/525259769ab0475a8ee2e97e39c4a803 2024-12-05T00:42:11,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/29d0769ed9194b76b34131947a16644d 2024-12-05T00:42:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/57505246e04e49c19106ffc0954e9afa 2024-12-05T00:42:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/572b83511a1045e29af2987e12d1f0e9 2024-12-05T00:42:11,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d86cc77a7ca14ca9a9bb5ca68bb14372 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d86cc77a7ca14ca9a9bb5ca68bb14372 2024-12-05T00:42:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/13161be3336845a0b676752af1d62e41 2024-12-05T00:42:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/642636cb5d8049d28596c76ec3b9fd2c 2024-12-05T00:42:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/10324e5db35844f7b6af16aa6107b1ff to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/10324e5db35844f7b6af16aa6107b1ff 2024-12-05T00:42:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/b427986c5d604defb2c49dd864aa19fd 2024-12-05T00:42:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/cd268e97407640ddb3ef9e6a5a093dc4 2024-12-05T00:42:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/34ac52c486e14c0fbf7fb0e5db11078b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/34ac52c486e14c0fbf7fb0e5db11078b 2024-12-05T00:42:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/62623e15bd164cc79509883a96e43d9f 2024-12-05T00:42:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/88e9a70fadfd4fb8bb8b350b04a9f9fa to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/88e9a70fadfd4fb8bb8b350b04a9f9fa 2024-12-05T00:42:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/95fb2dd9b670486594e0927228731c2a 2024-12-05T00:42:11,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/927f08050ad64a79b53a6af41a07f708 2024-12-05T00:42:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ee49c879fcec43a59c036afa5d3d400b 2024-12-05T00:42:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ef13f6e59f0b4444a7298edc447188c7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ef13f6e59f0b4444a7298edc447188c7 2024-12-05T00:42:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/4d146e755be744cca0a19a31967ec87c 2024-12-05T00:42:11,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/73ce3175be6648818260fac40e24b27c 2024-12-05T00:42:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/7749071694814e8ab642e0563bf57a9b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/7749071694814e8ab642e0563bf57a9b 2024-12-05T00:42:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/ec0a90f5a9134339a825cc8cb21d0c02 2024-12-05T00:42:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/bd344d878312405b969ddb80c05d88a8 2024-12-05T00:42:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/11b75747dd15402b8a1a340c3112937c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/b01d3b94170a422d924a44f2733a7976, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a301201d2bf34952bd208cfe849af291, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/e82163e373c4428ebdece49f94903e30, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c28ed701b717445ea285dfccae7d950b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/27ad0542cf2142c78def7435d9f9b21b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/78afc9f867de41999ed8daec70e82edb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0da1f8ccb8d340d9820edefe75b78e59, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/48a6926e1a9a402d9fc165be6baddfde, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/60037a2ab2534d12ba5696b3d047f310, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ddac260b332b4a70b9411d7f15d87844, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/9728828e37c34aabb60da22da7b5cd3d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/5e43a2cb519d4dd79bd65072b9a3701c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0ba8c38aab56466088f7011f2de06fcf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4376df782b794bf394aa8609cfab8689, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/46254b1145cc4d599cf336189b764613, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/fce0ae215de443e19258fb908dc3fbc6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/06ccce9dbec14e1aa39a6d2b5a55c6a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/d95debdb10394c92a70462c8914b5ec8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/50556429478843e7858f89ffaf427a1e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c0225de1b5e64c24a5c4b94c0d4036a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4ac2b4b2e7b84bd5a94ca10acb56a563, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/110f1662b3b0470aad099c5a566130f5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/98a134510f864d058261caa87d2a7dca, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/1f0487840a3e4f7db4d2baa3d2f5bd48] to archive 2024-12-05T00:42:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:42:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/11b75747dd15402b8a1a340c3112937c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/11b75747dd15402b8a1a340c3112937c 2024-12-05T00:42:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/b01d3b94170a422d924a44f2733a7976 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/b01d3b94170a422d924a44f2733a7976 2024-12-05T00:42:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a301201d2bf34952bd208cfe849af291 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a301201d2bf34952bd208cfe849af291 2024-12-05T00:42:11,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/e82163e373c4428ebdece49f94903e30 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/e82163e373c4428ebdece49f94903e30 2024-12-05T00:42:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c28ed701b717445ea285dfccae7d950b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c28ed701b717445ea285dfccae7d950b 2024-12-05T00:42:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/27ad0542cf2142c78def7435d9f9b21b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/27ad0542cf2142c78def7435d9f9b21b 2024-12-05T00:42:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/78afc9f867de41999ed8daec70e82edb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/78afc9f867de41999ed8daec70e82edb 2024-12-05T00:42:11,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0da1f8ccb8d340d9820edefe75b78e59 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0da1f8ccb8d340d9820edefe75b78e59 2024-12-05T00:42:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/48a6926e1a9a402d9fc165be6baddfde to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/48a6926e1a9a402d9fc165be6baddfde 2024-12-05T00:42:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/60037a2ab2534d12ba5696b3d047f310 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/60037a2ab2534d12ba5696b3d047f310 2024-12-05T00:42:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ddac260b332b4a70b9411d7f15d87844 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ddac260b332b4a70b9411d7f15d87844 2024-12-05T00:42:11,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/9728828e37c34aabb60da22da7b5cd3d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/9728828e37c34aabb60da22da7b5cd3d 2024-12-05T00:42:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/5e43a2cb519d4dd79bd65072b9a3701c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/5e43a2cb519d4dd79bd65072b9a3701c 2024-12-05T00:42:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0ba8c38aab56466088f7011f2de06fcf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/0ba8c38aab56466088f7011f2de06fcf 2024-12-05T00:42:11,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4376df782b794bf394aa8609cfab8689 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4376df782b794bf394aa8609cfab8689 2024-12-05T00:42:11,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/46254b1145cc4d599cf336189b764613 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/46254b1145cc4d599cf336189b764613 2024-12-05T00:42:11,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/fce0ae215de443e19258fb908dc3fbc6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/fce0ae215de443e19258fb908dc3fbc6 2024-12-05T00:42:11,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/06ccce9dbec14e1aa39a6d2b5a55c6a7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/06ccce9dbec14e1aa39a6d2b5a55c6a7 2024-12-05T00:42:11,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/d95debdb10394c92a70462c8914b5ec8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/d95debdb10394c92a70462c8914b5ec8 2024-12-05T00:42:11,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/50556429478843e7858f89ffaf427a1e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/50556429478843e7858f89ffaf427a1e 2024-12-05T00:42:11,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c0225de1b5e64c24a5c4b94c0d4036a7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/c0225de1b5e64c24a5c4b94c0d4036a7 2024-12-05T00:42:11,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4ac2b4b2e7b84bd5a94ca10acb56a563 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/4ac2b4b2e7b84bd5a94ca10acb56a563 2024-12-05T00:42:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/110f1662b3b0470aad099c5a566130f5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/110f1662b3b0470aad099c5a566130f5 2024-12-05T00:42:11,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/98a134510f864d058261caa87d2a7dca to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/98a134510f864d058261caa87d2a7dca 2024-12-05T00:42:11,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/1f0487840a3e4f7db4d2baa3d2f5bd48 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/1f0487840a3e4f7db4d2baa3d2f5bd48 2024-12-05T00:42:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/3e96a88ef8b14485aa9dd2676052f93a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/dac94c2b33504cab9baaefa568e2b94e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0a53c7af1e4d4091930d31ab14597461, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/77f8f67bae104ee4901a3304c43bfd4b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/4ccd832a80b1473bb36187d05060ada6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f2846c5c8c0c45bc8715390c7cb83c43, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0fbadf62836c49a99423b4ca0f588214, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/57e0205dfaec44e7a43e8db1af5c544f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6e6b3b859fe2489fad829c7e1fc3f049, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/23f10e0b09334629b10c31474bcfe1ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/60e75b2d7d9447d7b6a1abffff068ee4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/414b15c215fc41b08520f3963edc326a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/2afce1ae657e4150b21914259facf066, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cbb55aff6e734d2b8da83ff13747fadd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/68d1cfe2a13244c6aeca39474a148b50, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6c43712e27754979a7ad6f20e7d59ffa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/24f6b59c40a54de6aca5afb91fa46592, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/116994c52825402cb081db115be504be, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cad7c1025fe145699e6c70cb2ecd1545, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ef7bbf09ce4c4054aaa4f24a853541c7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f6076d7f4e1b464eac7e04010e1624ad, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/c2a3649198e74473a011661ba88ee060, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ffd74819511d4f09aa0c867792d90780, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/054f3fd8df324b19b516698a4f06f2cd] to archive 2024-12-05T00:42:11,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:42:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/3e96a88ef8b14485aa9dd2676052f93a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/3e96a88ef8b14485aa9dd2676052f93a 2024-12-05T00:42:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/dac94c2b33504cab9baaefa568e2b94e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/dac94c2b33504cab9baaefa568e2b94e 2024-12-05T00:42:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0a53c7af1e4d4091930d31ab14597461 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0a53c7af1e4d4091930d31ab14597461 2024-12-05T00:42:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/77f8f67bae104ee4901a3304c43bfd4b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/77f8f67bae104ee4901a3304c43bfd4b 2024-12-05T00:42:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/4ccd832a80b1473bb36187d05060ada6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/4ccd832a80b1473bb36187d05060ada6 2024-12-05T00:42:11,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f2846c5c8c0c45bc8715390c7cb83c43 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f2846c5c8c0c45bc8715390c7cb83c43 2024-12-05T00:42:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0fbadf62836c49a99423b4ca0f588214 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/0fbadf62836c49a99423b4ca0f588214 2024-12-05T00:42:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/57e0205dfaec44e7a43e8db1af5c544f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/57e0205dfaec44e7a43e8db1af5c544f 2024-12-05T00:42:11,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6e6b3b859fe2489fad829c7e1fc3f049 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6e6b3b859fe2489fad829c7e1fc3f049 2024-12-05T00:42:11,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/23f10e0b09334629b10c31474bcfe1ad to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/23f10e0b09334629b10c31474bcfe1ad 2024-12-05T00:42:11,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/60e75b2d7d9447d7b6a1abffff068ee4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/60e75b2d7d9447d7b6a1abffff068ee4 2024-12-05T00:42:11,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/b2ce295c3bcc4eb1bac0f5220bb4eb0c 2024-12-05T00:42:11,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/414b15c215fc41b08520f3963edc326a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/414b15c215fc41b08520f3963edc326a 2024-12-05T00:42:11,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/2afce1ae657e4150b21914259facf066 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/2afce1ae657e4150b21914259facf066 2024-12-05T00:42:11,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cbb55aff6e734d2b8da83ff13747fadd to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cbb55aff6e734d2b8da83ff13747fadd 2024-12-05T00:42:11,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/68d1cfe2a13244c6aeca39474a148b50 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/68d1cfe2a13244c6aeca39474a148b50 2024-12-05T00:42:11,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6c43712e27754979a7ad6f20e7d59ffa to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/6c43712e27754979a7ad6f20e7d59ffa 2024-12-05T00:42:11,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/24f6b59c40a54de6aca5afb91fa46592 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/24f6b59c40a54de6aca5afb91fa46592 2024-12-05T00:42:11,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/116994c52825402cb081db115be504be to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/116994c52825402cb081db115be504be 2024-12-05T00:42:12,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cad7c1025fe145699e6c70cb2ecd1545 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/cad7c1025fe145699e6c70cb2ecd1545 2024-12-05T00:42:12,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ef7bbf09ce4c4054aaa4f24a853541c7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ef7bbf09ce4c4054aaa4f24a853541c7 2024-12-05T00:42:12,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f6076d7f4e1b464eac7e04010e1624ad to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/f6076d7f4e1b464eac7e04010e1624ad 2024-12-05T00:42:12,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/c2a3649198e74473a011661ba88ee060 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/c2a3649198e74473a011661ba88ee060 2024-12-05T00:42:12,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ffd74819511d4f09aa0c867792d90780 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/ffd74819511d4f09aa0c867792d90780 2024-12-05T00:42:12,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/054f3fd8df324b19b516698a4f06f2cd to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/054f3fd8df324b19b516698a4f06f2cd 2024-12-05T00:42:12,009 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=4 2024-12-05T00:42:12,009 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. 2024-12-05T00:42:12,009 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for c39421bab8284c2f34cba4e4ea472def: Waiting for close lock at 1733359330235Running coprocessor pre-close hooks at 1733359330235Disabling compacts and flushes for region at 1733359330235Disabling writes for close at 1733359330235Obtaining lock to block concurrent updates at 1733359330235Preparing flush snapshotting stores in c39421bab8284c2f34cba4e4ea472def at 1733359330235Finished memstore snapshotting TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733359330236 (+1 ms)Flushing stores of TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def. at 1733359330237 (+1 ms)Flushing c39421bab8284c2f34cba4e4ea472def/A: creating writer at 1733359330237Flushing mob file c39421bab8284c2f34cba4e4ea472def/A: appending metadata at 1733359330247 (+10 ms)Flushing mob file c39421bab8284c2f34cba4e4ea472def/A: closing flushed file at 1733359330247Flushing c39421bab8284c2f34cba4e4ea472def/A: appending metadata at 1733359330665 (+418 ms)Flushing c39421bab8284c2f34cba4e4ea472def/A: closing flushed file at 1733359330665Flushing c39421bab8284c2f34cba4e4ea472def/B: creating writer at 1733359331080 (+415 ms)Flushing c39421bab8284c2f34cba4e4ea472def/B: appending metadata at 1733359331084 (+4 ms)Flushing c39421bab8284c2f34cba4e4ea472def/B: closing flushed file at 1733359331084Flushing c39421bab8284c2f34cba4e4ea472def/C: creating writer at 1733359331498 (+414 ms)Flushing c39421bab8284c2f34cba4e4ea472def/C: appending metadata at 1733359331503 (+5 ms)Flushing c39421bab8284c2f34cba4e4ea472def/C: closing flushed file at 1733359331503Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@212fce2: reopening flushed file at 1733359331913 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@176aaced: reopening flushed file at 1733359331919 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a452e0: reopening flushed file at 1733359331925 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c39421bab8284c2f34cba4e4ea472def in 1696ms, sequenceid=348, compaction requested=false at 1733359331931 (+6 ms)Writing region close event to WAL at 1733359332006 (+75 ms)Running coprocessor post-close hooks at 1733359332009 (+3 ms)Closed at 1733359332009 2024-12-05T00:42:12,011 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=c39421bab8284c2f34cba4e4ea472def, regionState=CLOSED 2024-12-05T00:42:12,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:42:12,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T00:42:12,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure c39421bab8284c2f34cba4e4ea472def, server=fea72ea5c4b6,42683,1733359267474 in 1.9320 sec 2024-12-05T00:42:12,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-05T00:42:12,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c39421bab8284c2f34cba4e4ea472def, UNASSIGN in 1.9370 sec 2024-12-05T00:42:12,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T00:42:12,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9410 sec 2024-12-05T00:42:12,021 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359332021"}]},"ts":"1733359332021"} 2024-12-05T00:42:12,023 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T00:42:12,023 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T00:42:12,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-12-05T00:42:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T00:42:12,223 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:12,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T00:42:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,227 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,228 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T00:42:12,230 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,235 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/recovered.edits] 2024-12-05T00:42:12,239 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d379d23d0b31491abe6fc4b9f86c2a03 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/d379d23d0b31491abe6fc4b9f86c2a03 2024-12-05T00:42:12,240 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/e30b8b3cb82640daac5f94dc4f8a0e29 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/A/e30b8b3cb82640daac5f94dc4f8a0e29 2024-12-05T00:42:12,244 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a39bcda9f60941788e4a356436293250 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/a39bcda9f60941788e4a356436293250 2024-12-05T00:42:12,246 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ba9f7a5339b34345af9fbb68cec2e200 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/B/ba9f7a5339b34345af9fbb68cec2e200 2024-12-05T00:42:12,250 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/55214cfe5a97472c91eb3a9e7789a27a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/55214cfe5a97472c91eb3a9e7789a27a 2024-12-05T00:42:12,252 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/df6018b163f24b3f8867725ea46bb7ab to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/C/df6018b163f24b3f8867725ea46bb7ab 2024-12-05T00:42:12,256 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/recovered.edits/351.seqid to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def/recovered.edits/351.seqid 2024-12-05T00:42:12,256 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,257 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T00:42:12,257 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T00:42:12,258 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T00:42:12,263 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205007c7099e493465d88eb69dac374321c_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205007c7099e493465d88eb69dac374321c_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,264 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205022e6b2098484448817ace39d0442c3c_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205022e6b2098484448817ace39d0442c3c_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,266 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050f49bccb770842f3997bbe102ad4c625_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050f49bccb770842f3997bbe102ad4c625_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,267 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051d555f24de634ff1b90a8c9064553609_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051d555f24de634ff1b90a8c9064553609_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052610f06964354dc68cfab4c1843d5195_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052610f06964354dc68cfab4c1843d5195_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,270 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053128122070d442a9a47fa5a038ec0929_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053128122070d442a9a47fa5a038ec0929_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,271 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a55f07d39364694b0f118bd71688468_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a55f07d39364694b0f118bd71688468_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,272 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120545563c243b0b4a1f862e87176a5d2ecc_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120545563c243b0b4a1f862e87176a5d2ecc_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,274 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120546592847324a4cd2af6cf3fa1ebd7438_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120546592847324a4cd2af6cf3fa1ebd7438_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,275 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054fede699d970498ba52a80987acfd8ee_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054fede699d970498ba52a80987acfd8ee_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,276 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056888e933ade141d3b44f4f783b100d00_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056888e933ade141d3b44f4f783b100d00_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056ec831ed4f6f4c93ba6641d355eafcb6_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056ec831ed4f6f4c93ba6641d355eafcb6_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,278 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058dd1f8049feb4193afc849d01107c461_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058dd1f8049feb4193afc849d01107c461_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,279 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059d0ae782f7684f429a91a2b7e0cebaa5_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059d0ae782f7684f429a91a2b7e0cebaa5_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,280 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205adab03c701624e97a2b4b01a69f7aa3d_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205adab03c701624e97a2b4b01a69f7aa3d_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,282 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b1c36f43596f46c2867289283c1ef246_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b1c36f43596f46c2867289283c1ef246_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,283 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cca7f5a3af414737aaccedef91990c75_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cca7f5a3af414737aaccedef91990c75_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,284 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cd91bbcfaeb14b30be59feeaa0c3d4b3_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cd91bbcfaeb14b30be59feeaa0c3d4b3_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,286 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ebf3025096dc40e88a47deadd4ab404e_c39421bab8284c2f34cba4e4ea472def to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ebf3025096dc40e88a47deadd4ab404e_c39421bab8284c2f34cba4e4ea472def 2024-12-05T00:42:12,286 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T00:42:12,290 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,292 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T00:42:12,294 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T00:42:12,296 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,296 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T00:42:12,296 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733359332296"}]},"ts":"9223372036854775807"} 2024-12-05T00:42:12,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T00:42:12,299 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c39421bab8284c2f34cba4e4ea472def, NAME => 'TestAcidGuarantees,,1733359300097.c39421bab8284c2f34cba4e4ea472def.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T00:42:12,299 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T00:42:12,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733359332299"}]},"ts":"9223372036854775807"} 2024-12-05T00:42:12,302 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T00:42:12,302 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-12-05T00:42:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T00:42:12,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T00:42:12,332 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:12,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=9218 (was 9464) 2024-12-05T00:42:12,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=329, ProcessCount=11, AvailableMemoryMB=9218 2024-12-05T00:42:12,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:42:12,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:42:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:12,359 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:42:12,359 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:12,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-05T00:42:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T00:42:12,361 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:42:12,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742056_1232 (size=1061) 2024-12-05T00:42:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T00:42:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T00:42:12,771 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:42:12,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742057_1233 (size=53) 2024-12-05T00:42:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T00:42:13,182 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:42:13,182 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8052c0b234f27c5e419e64636802bdde, disabling compactions & flushes 2024-12-05T00:42:13,182 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,182 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,182 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. after waiting 0 ms 2024-12-05T00:42:13,182 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,182 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,183 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8052c0b234f27c5e419e64636802bdde: Waiting for close lock at 1733359333182Disabling compacts and flushes for region at 1733359333182Disabling writes for close at 1733359333182Writing region close event to WAL at 1733359333182Closed at 1733359333182 2024-12-05T00:42:13,185 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:42:13,186 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733359333185"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359333185"}]},"ts":"1733359333185"} 2024-12-05T00:42:13,189 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:42:13,191 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:42:13,192 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359333191"}]},"ts":"1733359333191"} 2024-12-05T00:42:13,195 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T00:42:13,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, ASSIGN}] 2024-12-05T00:42:13,197 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, ASSIGN 2024-12-05T00:42:13,197 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=false 2024-12-05T00:42:13,348 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8052c0b234f27c5e419e64636802bdde, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:13,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, ASSIGN because future has completed 2024-12-05T00:42:13,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:42:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T00:42:13,505 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,506 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,508 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:13,508 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8052c0b234f27c5e419e64636802bdde columnFamilyName A 2024-12-05T00:42:13,508 DEBUG [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:13,508 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.HStore(327): Store=8052c0b234f27c5e419e64636802bdde/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:13,508 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,510 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:13,510 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8052c0b234f27c5e419e64636802bdde columnFamilyName B 2024-12-05T00:42:13,510 DEBUG [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:13,510 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.HStore(327): Store=8052c0b234f27c5e419e64636802bdde/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:13,510 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,511 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:13,511 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8052c0b234f27c5e419e64636802bdde columnFamilyName C 2024-12-05T00:42:13,511 DEBUG [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:13,512 INFO [StoreOpener-8052c0b234f27c5e419e64636802bdde-1 {}] regionserver.HStore(327): Store=8052c0b234f27c5e419e64636802bdde/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:13,512 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,512 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,513 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,513 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,514 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,514 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,515 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:42:13,516 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,518 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:42:13,519 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 8052c0b234f27c5e419e64636802bdde; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67836587, jitterRate=0.010843917727470398}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:42:13,519 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:13,519 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 8052c0b234f27c5e419e64636802bdde: Running coprocessor pre-open hook at 1733359333505Writing region info on filesystem at 1733359333505Initializing all the Stores at 1733359333506 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359333506Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359333506Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359333506Cleaning up temporary data from old regions at 1733359333514 (+8 ms)Running coprocessor post-open hooks at 1733359333519 (+5 ms)Region opened successfully at 1733359333519 2024-12-05T00:42:13,520 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., pid=65, masterSystemTime=1733359333502 2024-12-05T00:42:13,522 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,523 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:13,523 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8052c0b234f27c5e419e64636802bdde, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:13,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:42:13,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-05T00:42:13,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 in 176 msec 2024-12-05T00:42:13,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-05T00:42:13,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, ASSIGN in 333 msec 2024-12-05T00:42:13,531 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:42:13,532 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359333531"}]},"ts":"1733359333531"} 2024-12-05T00:42:13,534 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T00:42:13,535 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:42:13,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-05T00:42:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-05T00:42:14,512 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:14,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79eacb52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,514 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6f92b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743412a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76a17bbf 2024-12-05T00:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:42:14,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:42:14,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a054597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,525 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6323ec97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,528 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b839647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d1dc57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,530 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db372bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e50bd94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a525a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,534 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687de245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2022fc7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c2c082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,539 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7139a906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,541 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610dee85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29df300d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,543 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6173fc85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,546 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f10e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c38859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,549 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234a8919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c643f24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3197c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf0f80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8ab030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533cbeb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206e8ff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76989242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a14518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,563 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d6af946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,564 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11efdcf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef84e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:14,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:14,567 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d90ae14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:14,568 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c727e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:14,569 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,569 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:14,573 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,573 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-05T00:42:14,574 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T00:42:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,574 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,575 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,576 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,577 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:14,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:14,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:14,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:42:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:14,584 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,606 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/c582d8e0fe404440995682927a3e8f97 is 50, key is test_row_0/A:col10/1733359334581/Put/seqid=0 2024-12-05T00:42:14,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359394611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,616 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:14,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359394612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:14,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359394613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359394614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:42:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359394614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742058_1234 (size=14341) 2024-12-05T00:42:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T00:42:14,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359394722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359394722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359394722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359394732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359394732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T00:42:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:14,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:14,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T00:42:14,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:14,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:14,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:14,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:14,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T00:42:14,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359394931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359394932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359394932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359394942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359394942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/c582d8e0fe404440995682927a3e8f97 2024-12-05T00:42:15,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T00:42:15,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:15,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:15,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:15,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:15,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:15,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9c3638b8df6e4d5781d52b6cf9980f52 is 50, key is test_row_0/B:col10/1733359334581/Put/seqid=0 2024-12-05T00:42:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742059_1235 (size=12001) 2024-12-05T00:42:15,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9c3638b8df6e4d5781d52b6cf9980f52 2024-12-05T00:42:15,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/3e40c85731cc4090a608150d07627439 is 50, key is test_row_0/C:col10/1733359334581/Put/seqid=0 2024-12-05T00:42:15,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742060_1236 (size=12001) 2024-12-05T00:42:15,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/3e40c85731cc4090a608150d07627439 2024-12-05T00:42:15,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/c582d8e0fe404440995682927a3e8f97 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c582d8e0fe404440995682927a3e8f97 2024-12-05T00:42:15,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c582d8e0fe404440995682927a3e8f97, entries=200, sequenceid=13, filesize=14.0 K 2024-12-05T00:42:15,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9c3638b8df6e4d5781d52b6cf9980f52 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9c3638b8df6e4d5781d52b6cf9980f52 2024-12-05T00:42:15,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9c3638b8df6e4d5781d52b6cf9980f52, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T00:42:15,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/3e40c85731cc4090a608150d07627439 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3e40c85731cc4090a608150d07627439 2024-12-05T00:42:15,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3e40c85731cc4090a608150d07627439, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T00:42:15,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8052c0b234f27c5e419e64636802bdde in 594ms, sequenceid=13, compaction requested=false 2024-12-05T00:42:15,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:15,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T00:42:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:15,199 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:42:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T00:42:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/9d2ead3fe4ad4044a3659f28c46ba8e4 is 50, key is test_row_0/A:col10/1733359334613/Put/seqid=0 2024-12-05T00:42:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:15,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:15,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742061_1237 (size=12001) 2024-12-05T00:42:15,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/9d2ead3fe4ad4044a3659f28c46ba8e4 2024-12-05T00:42:15,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359395256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359395256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359395261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359395265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359395262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c3fe635ac8e74d34b8935581858edadf is 50, key is test_row_0/B:col10/1733359334613/Put/seqid=0 2024-12-05T00:42:15,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742062_1238 (size=12001) 2024-12-05T00:42:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359395371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359395371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359395382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359395382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359395382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359395581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359395582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359395592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359395592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359395592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T00:42:15,755 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c3fe635ac8e74d34b8935581858edadf 2024-12-05T00:42:15,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/65025a84422c4bfe99f047effca91b2a is 50, key is test_row_0/C:col10/1733359334613/Put/seqid=0 2024-12-05T00:42:15,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742063_1239 (size=12001) 2024-12-05T00:42:15,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359395891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359395891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359395902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359395902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359395902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,170 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/65025a84422c4bfe99f047effca91b2a 2024-12-05T00:42:16,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/9d2ead3fe4ad4044a3659f28c46ba8e4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/9d2ead3fe4ad4044a3659f28c46ba8e4 2024-12-05T00:42:16,179 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/9d2ead3fe4ad4044a3659f28c46ba8e4, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T00:42:16,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c3fe635ac8e74d34b8935581858edadf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c3fe635ac8e74d34b8935581858edadf 2024-12-05T00:42:16,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c3fe635ac8e74d34b8935581858edadf, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T00:42:16,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/65025a84422c4bfe99f047effca91b2a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/65025a84422c4bfe99f047effca91b2a 2024-12-05T00:42:16,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/65025a84422c4bfe99f047effca91b2a, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T00:42:16,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8052c0b234f27c5e419e64636802bdde in 994ms, sequenceid=33, compaction requested=false 2024-12-05T00:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-05T00:42:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-05T00:42:16,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-05T00:42:16,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-12-05T00:42:16,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.6250 sec 2024-12-05T00:42:16,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:42:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:16,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/be911a0ba679469da1c0fd49957ecec0 is 50, key is test_row_0/A:col10/1733359336402/Put/seqid=0 2024-12-05T00:42:16,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359396431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359396436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359396437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359396439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359396439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742064_1240 (size=14341) 2024-12-05T00:42:16,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/be911a0ba679469da1c0fd49957ecec0 2024-12-05T00:42:16,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c222781b3492430fa5eaa96da28d4a87 is 50, key is test_row_0/B:col10/1733359336402/Put/seqid=0 2024-12-05T00:42:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742065_1241 (size=12001) 2024-12-05T00:42:16,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c222781b3492430fa5eaa96da28d4a87 2024-12-05T00:42:16,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/64d50dff3f234b6789c688194f7cd576 is 50, key is test_row_0/C:col10/1733359336402/Put/seqid=0 2024-12-05T00:42:16,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359396542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742066_1242 (size=12001) 2024-12-05T00:42:16,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/64d50dff3f234b6789c688194f7cd576 2024-12-05T00:42:16,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359396552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359396552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359396553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359396553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/be911a0ba679469da1c0fd49957ecec0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/be911a0ba679469da1c0fd49957ecec0 2024-12-05T00:42:16,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/be911a0ba679469da1c0fd49957ecec0, entries=200, sequenceid=46, filesize=14.0 K 2024-12-05T00:42:16,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c222781b3492430fa5eaa96da28d4a87 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c222781b3492430fa5eaa96da28d4a87 2024-12-05T00:42:16,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c222781b3492430fa5eaa96da28d4a87, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T00:42:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/64d50dff3f234b6789c688194f7cd576 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/64d50dff3f234b6789c688194f7cd576 2024-12-05T00:42:16,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/64d50dff3f234b6789c688194f7cd576, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T00:42:16,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8052c0b234f27c5e419e64636802bdde in 189ms, sequenceid=46, compaction requested=true 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:16,592 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:16,592 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:16,595 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:16,595 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:16,595 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:16,595 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c582d8e0fe404440995682927a3e8f97, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/9d2ead3fe4ad4044a3659f28c46ba8e4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/be911a0ba679469da1c0fd49957ecec0] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=39.7 K 2024-12-05T00:42:16,596 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:16,596 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c582d8e0fe404440995682927a3e8f97, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733359334580 2024-12-05T00:42:16,596 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:16,596 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:16,596 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9c3638b8df6e4d5781d52b6cf9980f52, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c3fe635ac8e74d34b8935581858edadf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c222781b3492430fa5eaa96da28d4a87] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=35.2 K 2024-12-05T00:42:16,597 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3638b8df6e4d5781d52b6cf9980f52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733359334581 2024-12-05T00:42:16,598 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2ead3fe4ad4044a3659f28c46ba8e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733359334610 2024-12-05T00:42:16,598 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3fe635ac8e74d34b8935581858edadf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733359334610 2024-12-05T00:42:16,598 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting be911a0ba679469da1c0fd49957ecec0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359335245 2024-12-05T00:42:16,599 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c222781b3492430fa5eaa96da28d4a87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359335245 2024-12-05T00:42:16,635 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:16,635 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0dab45a336ab4ce3bee05bdcb50f14b6 is 50, key is test_row_0/B:col10/1733359336402/Put/seqid=0 2024-12-05T00:42:16,666 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:16,667 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/10b4d9a8098a4cf9b4526910f2daf602 is 50, key is test_row_0/A:col10/1733359336402/Put/seqid=0 2024-12-05T00:42:16,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742067_1243 (size=12104) 2024-12-05T00:42:16,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742068_1244 (size=12104) 2024-12-05T00:42:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T00:42:16,722 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:16,723 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-05T00:42:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T00:42:16,725 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:16,726 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:16,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:16,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359336431, max timestamp=1733359336755Num uniques 859; 2024-12-05T00:42:16,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359336431, max timestamp=1733359336755Num uniques 859; 2024-12-05T00:42:16,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:16,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:16,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359336431, max timestamp=1733359336755Num uniques 859; 2024-12-05T00:42:16,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:16,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-05T00:42:16,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:16,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:16,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:16,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:16,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:16,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:16,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/4057f8748c154477a835db6e37af97b9 is 50, key is test_row_0/A:col10/1733359336765/Put/seqid=0 2024-12-05T00:42:16,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359336790, max timestamp=1733359336806Num uniques 859; 2024-12-05T00:42:16,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:16,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742069_1245 (size=12001) 2024-12-05T00:42:16,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/4057f8748c154477a835db6e37af97b9 2024-12-05T00:42:16,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359336790, max timestamp=1733359336806Num uniques 859; 2024-12-05T00:42:16,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:16,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359336790, max timestamp=1733359336806Num uniques 859; 2024-12-05T00:42:16,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:16,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0fd4c6b19c66462cbc8636f6631d2ca0 is 50, key is test_row_0/B:col10/1733359336765/Put/seqid=0 2024-12-05T00:42:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T00:42:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359396836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359396837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359396842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359396844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359396844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T00:42:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:16,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:16,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742070_1246 (size=12001) 2024-12-05T00:42:16,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359396952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359396952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359396961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359396961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:16,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359396962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T00:42:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:17,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T00:42:17,098 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0dab45a336ab4ce3bee05bdcb50f14b6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0dab45a336ab4ce3bee05bdcb50f14b6 2024-12-05T00:42:17,109 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into 0dab45a336ab4ce3bee05bdcb50f14b6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:17,109 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:17,109 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359336592; duration=0sec 2024-12-05T00:42:17,109 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:17,109 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:17,109 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:17,112 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:17,112 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:17,112 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,113 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3e40c85731cc4090a608150d07627439, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/65025a84422c4bfe99f047effca91b2a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/64d50dff3f234b6789c688194f7cd576] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=35.2 K 2024-12-05T00:42:17,113 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e40c85731cc4090a608150d07627439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733359334581 2024-12-05T00:42:17,113 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65025a84422c4bfe99f047effca91b2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733359334610 2024-12-05T00:42:17,113 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d50dff3f234b6789c688194f7cd576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359335245 2024-12-05T00:42:17,116 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/10b4d9a8098a4cf9b4526910f2daf602 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/10b4d9a8098a4cf9b4526910f2daf602 2024-12-05T00:42:17,121 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 10b4d9a8098a4cf9b4526910f2daf602(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:17,121 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:17,121 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359336592; duration=0sec 2024-12-05T00:42:17,121 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:17,121 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:17,123 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:17,123 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/02d95c418fec4770ab91e45eaf1bee4b is 50, key is test_row_0/C:col10/1733359336402/Put/seqid=0 2024-12-05T00:42:17,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742071_1247 (size=12104) 2024-12-05T00:42:17,145 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/02d95c418fec4770ab91e45eaf1bee4b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/02d95c418fec4770ab91e45eaf1bee4b 2024-12-05T00:42:17,152 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into 02d95c418fec4770ab91e45eaf1bee4b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:17,153 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:17,153 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359336592; duration=0sec 2024-12-05T00:42:17,153 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:17,153 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359397161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359397161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359397172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359397172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359397172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T00:42:17,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:17,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:17,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:17,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0fd4c6b19c66462cbc8636f6631d2ca0 2024-12-05T00:42:17,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/4cde91b2360e499688fd29edfac75494 is 50, key is test_row_0/C:col10/1733359336765/Put/seqid=0 2024-12-05T00:42:17,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742072_1248 (size=12001) 2024-12-05T00:42:17,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/4cde91b2360e499688fd29edfac75494 2024-12-05T00:42:17,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/4057f8748c154477a835db6e37af97b9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/4057f8748c154477a835db6e37af97b9 2024-12-05T00:42:17,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/4057f8748c154477a835db6e37af97b9, entries=150, sequenceid=71, filesize=11.7 K 2024-12-05T00:42:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0fd4c6b19c66462cbc8636f6631d2ca0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0fd4c6b19c66462cbc8636f6631d2ca0 2024-12-05T00:42:17,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0fd4c6b19c66462cbc8636f6631d2ca0, entries=150, sequenceid=71, filesize=11.7 K 2024-12-05T00:42:17,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/4cde91b2360e499688fd29edfac75494 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/4cde91b2360e499688fd29edfac75494 2024-12-05T00:42:17,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/4cde91b2360e499688fd29edfac75494, entries=150, sequenceid=71, filesize=11.7 K 2024-12-05T00:42:17,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 8052c0b234f27c5e419e64636802bdde in 551ms, sequenceid=71, compaction requested=false 2024-12-05T00:42:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:17,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T00:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,339 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T00:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:17,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:17,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:17,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:17,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/75ccd82bfb8542dba754bbab2a07bdf0 is 50, key is test_row_0/A:col10/1733359336835/Put/seqid=0 2024-12-05T00:42:17,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742073_1249 (size=12001) 2024-12-05T00:42:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T00:42:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359397481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359397481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359397481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359397482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359397484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,534 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:42:17,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359397592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359397592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,749 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/75ccd82bfb8542dba754bbab2a07bdf0 2024-12-05T00:42:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f1272edd5c5846e485f3ab9de619b612 is 50, key is test_row_0/B:col10/1733359336835/Put/seqid=0 2024-12-05T00:42:17,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742074_1250 (size=12001) 2024-12-05T00:42:17,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f1272edd5c5846e485f3ab9de619b612 2024-12-05T00:42:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/dbf950f6ebd74564898827acf2ac702f is 50, key is test_row_0/C:col10/1733359336835/Put/seqid=0 2024-12-05T00:42:17,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742075_1251 (size=12001) 2024-12-05T00:42:17,803 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/dbf950f6ebd74564898827acf2ac702f 2024-12-05T00:42:17,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359397802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359397803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:17,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/75ccd82bfb8542dba754bbab2a07bdf0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/75ccd82bfb8542dba754bbab2a07bdf0 2024-12-05T00:42:17,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/75ccd82bfb8542dba754bbab2a07bdf0, entries=150, sequenceid=111, filesize=11.7 K 2024-12-05T00:42:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f1272edd5c5846e485f3ab9de619b612 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f1272edd5c5846e485f3ab9de619b612 2024-12-05T00:42:17,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f1272edd5c5846e485f3ab9de619b612, entries=150, sequenceid=111, filesize=11.7 K 2024-12-05T00:42:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/dbf950f6ebd74564898827acf2ac702f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/dbf950f6ebd74564898827acf2ac702f 2024-12-05T00:42:17,842 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/dbf950f6ebd74564898827acf2ac702f, entries=150, sequenceid=111, filesize=11.7 K 2024-12-05T00:42:17,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8052c0b234f27c5e419e64636802bdde in 504ms, sequenceid=111, compaction requested=true 2024-12-05T00:42:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-05T00:42:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-05T00:42:17,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T00:42:17,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1190 sec 2024-12-05T00:42:17,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1250 sec 2024-12-05T00:42:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T00:42:17,862 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:17,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-05T00:42:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:17,865 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:17,866 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:17,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:17,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:42:17,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:17,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:17,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:17,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:17,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:17,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:18,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/d45f46b713704d3d9605cf82e9c59e3a is 50, key is test_row_0/A:col10/1733359337993/Put/seqid=0 2024-12-05T00:42:18,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742076_1252 (size=12001) 2024-12-05T00:42:18,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/d45f46b713704d3d9605cf82e9c59e3a 2024-12-05T00:42:18,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/147db0ff33d741ee9db7adb224949fbf is 50, key is test_row_0/B:col10/1733359337993/Put/seqid=0 2024-12-05T00:42:18,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359398042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359398048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359398051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742077_1253 (size=12001) 2024-12-05T00:42:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359398162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359398162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359398162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:18,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359398372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359398372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359398372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/147db0ff33d741ee9db7adb224949fbf 2024-12-05T00:42:18,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/15c6bec1f05d49d4bed93c31477cf0f2 is 50, key is test_row_0/C:col10/1733359337993/Put/seqid=0 2024-12-05T00:42:18,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:18,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742078_1254 (size=12001) 2024-12-05T00:42:18,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/15c6bec1f05d49d4bed93c31477cf0f2 2024-12-05T00:42:18,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/d45f46b713704d3d9605cf82e9c59e3a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/d45f46b713704d3d9605cf82e9c59e3a 2024-12-05T00:42:18,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/d45f46b713704d3d9605cf82e9c59e3a, entries=150, sequenceid=123, filesize=11.7 K 2024-12-05T00:42:18,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/147db0ff33d741ee9db7adb224949fbf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/147db0ff33d741ee9db7adb224949fbf 2024-12-05T00:42:18,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/147db0ff33d741ee9db7adb224949fbf, entries=150, sequenceid=123, filesize=11.7 K 2024-12-05T00:42:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/15c6bec1f05d49d4bed93c31477cf0f2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/15c6bec1f05d49d4bed93c31477cf0f2 2024-12-05T00:42:18,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/15c6bec1f05d49d4bed93c31477cf0f2, entries=150, sequenceid=123, filesize=11.7 K 2024-12-05T00:42:18,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8052c0b234f27c5e419e64636802bdde in 532ms, sequenceid=123, compaction requested=true 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:18,526 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:18,526 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:18,527 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:18,528 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:18,528 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,528 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/10b4d9a8098a4cf9b4526910f2daf602, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/4057f8748c154477a835db6e37af97b9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/75ccd82bfb8542dba754bbab2a07bdf0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/d45f46b713704d3d9605cf82e9c59e3a] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=47.0 K 2024-12-05T00:42:18,528 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:18,528 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:18,528 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,528 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0dab45a336ab4ce3bee05bdcb50f14b6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0fd4c6b19c66462cbc8636f6631d2ca0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f1272edd5c5846e485f3ab9de619b612, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/147db0ff33d741ee9db7adb224949fbf] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=47.0 K 2024-12-05T00:42:18,529 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b4d9a8098a4cf9b4526910f2daf602, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359335245 2024-12-05T00:42:18,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dab45a336ab4ce3bee05bdcb50f14b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359335245 2024-12-05T00:42:18,529 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4057f8748c154477a835db6e37af97b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733359336436 2024-12-05T00:42:18,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fd4c6b19c66462cbc8636f6631d2ca0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733359336436 2024-12-05T00:42:18,529 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting f1272edd5c5846e485f3ab9de619b612, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733359336835 2024-12-05T00:42:18,530 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75ccd82bfb8542dba754bbab2a07bdf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733359336835 2024-12-05T00:42:18,530 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 147db0ff33d741ee9db7adb224949fbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733359337474 2024-12-05T00:42:18,530 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting d45f46b713704d3d9605cf82e9c59e3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733359337474 2024-12-05T00:42:18,540 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:18,540 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/bd3b1b23b5114dfbbc2fa5dd36a933b7 is 50, key is test_row_0/B:col10/1733359337993/Put/seqid=0 2024-12-05T00:42:18,543 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:18,543 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/bb4a161202ff4b73a117c77e7ec14059 is 50, key is test_row_0/A:col10/1733359337993/Put/seqid=0 2024-12-05T00:42:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742079_1255 (size=12241) 2024-12-05T00:42:18,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742080_1256 (size=12241) 2024-12-05T00:42:18,555 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/bd3b1b23b5114dfbbc2fa5dd36a933b7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bd3b1b23b5114dfbbc2fa5dd36a933b7 2024-12-05T00:42:18,563 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into bd3b1b23b5114dfbbc2fa5dd36a933b7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:18,563 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:18,563 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=12, startTime=1733359338526; duration=0sec 2024-12-05T00:42:18,564 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:18,564 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:18,564 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:18,566 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:18,566 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:18,566 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,566 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/02d95c418fec4770ab91e45eaf1bee4b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/4cde91b2360e499688fd29edfac75494, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/dbf950f6ebd74564898827acf2ac702f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/15c6bec1f05d49d4bed93c31477cf0f2] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=47.0 K 2024-12-05T00:42:18,567 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d95c418fec4770ab91e45eaf1bee4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359335245 2024-12-05T00:42:18,567 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cde91b2360e499688fd29edfac75494, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733359336436 2024-12-05T00:42:18,568 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf950f6ebd74564898827acf2ac702f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733359336835 2024-12-05T00:42:18,569 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c6bec1f05d49d4bed93c31477cf0f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733359337474 2024-12-05T00:42:18,577 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:18,578 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/3a184bc984134c6a9fe8874f366ae69c is 50, key is test_row_0/C:col10/1733359337993/Put/seqid=0 2024-12-05T00:42:18,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742081_1257 (size=12241) 2024-12-05T00:42:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:18,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T00:42:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:18,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/ff0a051d0355497fab01bccfa4227562 is 50, key is test_row_0/A:col10/1733359338633/Put/seqid=0 2024-12-05T00:42:18,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742082_1258 (size=14541) 2024-12-05T00:42:18,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/ff0a051d0355497fab01bccfa4227562 2024-12-05T00:42:18,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359398663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/77a6b01da7f84659ac19bcd2359f1c95 is 50, key is test_row_0/B:col10/1733359338633/Put/seqid=0 2024-12-05T00:42:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359398663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742083_1259 (size=12151) 2024-12-05T00:42:18,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359398681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/77a6b01da7f84659ac19bcd2359f1c95 2024-12-05T00:42:18,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359398681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359398682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/449abbb6328e488a9e562e792be51d70 is 50, key is test_row_0/C:col10/1733359338633/Put/seqid=0 2024-12-05T00:42:18,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742084_1260 (size=12151) 2024-12-05T00:42:18,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/449abbb6328e488a9e562e792be51d70 2024-12-05T00:42:18,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/ff0a051d0355497fab01bccfa4227562 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/ff0a051d0355497fab01bccfa4227562 2024-12-05T00:42:18,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/ff0a051d0355497fab01bccfa4227562, entries=200, sequenceid=145, filesize=14.2 K 2024-12-05T00:42:18,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/77a6b01da7f84659ac19bcd2359f1c95 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/77a6b01da7f84659ac19bcd2359f1c95 2024-12-05T00:42:18,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/77a6b01da7f84659ac19bcd2359f1c95, entries=150, sequenceid=145, filesize=11.9 K 2024-12-05T00:42:18,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/449abbb6328e488a9e562e792be51d70 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/449abbb6328e488a9e562e792be51d70 2024-12-05T00:42:18,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/449abbb6328e488a9e562e792be51d70, entries=150, sequenceid=145, filesize=11.9 K 2024-12-05T00:42:18,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8052c0b234f27c5e419e64636802bdde in 99ms, sequenceid=145, compaction requested=false 2024-12-05T00:42:18,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:18,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:42:18,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:18,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:18,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:18,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:18,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:18,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:18,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/a6e6e53ef0054dc3b94f411163af7316 is 50, key is test_row_0/A:col10/1733359338772/Put/seqid=0 2024-12-05T00:42:18,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742085_1261 (size=14541) 2024-12-05T00:42:18,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359398835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359398834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359398941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359398941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:18,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:18,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:18,955 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/bb4a161202ff4b73a117c77e7ec14059 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bb4a161202ff4b73a117c77e7ec14059 2024-12-05T00:42:18,960 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into bb4a161202ff4b73a117c77e7ec14059(size=12.0 K), total size for store is 26.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:18,960 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:18,960 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=12, startTime=1733359338526; duration=0sec 2024-12-05T00:42:18,960 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:18,960 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:18,994 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/3a184bc984134c6a9fe8874f366ae69c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3a184bc984134c6a9fe8874f366ae69c 2024-12-05T00:42:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:19,009 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into 3a184bc984134c6a9fe8874f366ae69c(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:19,010 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:19,010 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=12, startTime=1733359338526; duration=0sec 2024-12-05T00:42:19,010 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:19,010 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:19,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:19,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:19,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359399152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359399152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/a6e6e53ef0054dc3b94f411163af7316 2024-12-05T00:42:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359399192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359399192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359399192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/dbea2e8e8c1d404fa028a14eb83c0268 is 50, key is test_row_0/B:col10/1733359338772/Put/seqid=0 2024-12-05T00:42:19,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742086_1262 (size=12151) 2024-12-05T00:42:19,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359399461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359399468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:19,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:19,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/dbea2e8e8c1d404fa028a14eb83c0268 2024-12-05T00:42:19,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/675da4d58b5e46268d407111adcde812 is 50, key is test_row_0/C:col10/1733359338772/Put/seqid=0 2024-12-05T00:42:19,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742087_1263 (size=12151) 2024-12-05T00:42:19,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:19,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:19,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:19,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:19,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359399972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359399972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:20,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:20,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:20,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:20,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:20,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/675da4d58b5e46268d407111adcde812 2024-12-05T00:42:20,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/a6e6e53ef0054dc3b94f411163af7316 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a6e6e53ef0054dc3b94f411163af7316 2024-12-05T00:42:20,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a6e6e53ef0054dc3b94f411163af7316, entries=200, sequenceid=158, filesize=14.2 K 2024-12-05T00:42:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/dbea2e8e8c1d404fa028a14eb83c0268 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/dbea2e8e8c1d404fa028a14eb83c0268 2024-12-05T00:42:20,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/dbea2e8e8c1d404fa028a14eb83c0268, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T00:42:20,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/675da4d58b5e46268d407111adcde812 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/675da4d58b5e46268d407111adcde812 2024-12-05T00:42:20,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/675da4d58b5e46268d407111adcde812, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T00:42:20,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8052c0b234f27c5e419e64636802bdde in 1298ms, sequenceid=158, compaction requested=true 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:20,071 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:20,071 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:20,072 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:20,073 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:20,073 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,073 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bd3b1b23b5114dfbbc2fa5dd36a933b7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/77a6b01da7f84659ac19bcd2359f1c95, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/dbea2e8e8c1d404fa028a14eb83c0268] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=35.7 K 2024-12-05T00:42:20,073 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:20,073 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:20,073 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,073 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bb4a161202ff4b73a117c77e7ec14059, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/ff0a051d0355497fab01bccfa4227562, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a6e6e53ef0054dc3b94f411163af7316] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=40.4 K 2024-12-05T00:42:20,074 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb4a161202ff4b73a117c77e7ec14059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733359337474 2024-12-05T00:42:20,074 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3b1b23b5114dfbbc2fa5dd36a933b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733359337474 2024-12-05T00:42:20,074 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff0a051d0355497fab01bccfa4227562, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733359338038 2024-12-05T00:42:20,074 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a6b01da7f84659ac19bcd2359f1c95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733359338040 2024-12-05T00:42:20,075 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6e6e53ef0054dc3b94f411163af7316, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733359338660 2024-12-05T00:42:20,075 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting dbea2e8e8c1d404fa028a14eb83c0268, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733359338661 2024-12-05T00:42:20,084 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:20,084 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/e7e4ac0fad0443b1b04ef6c7645cd839 is 50, key is test_row_0/B:col10/1733359338772/Put/seqid=0 2024-12-05T00:42:20,089 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:20,089 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/639430b1ec4e4715a444ed552dd73fd2 is 50, key is test_row_0/A:col10/1733359338772/Put/seqid=0 2024-12-05T00:42:20,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742088_1264 (size=12493) 2024-12-05T00:42:20,107 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/e7e4ac0fad0443b1b04ef6c7645cd839 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/e7e4ac0fad0443b1b04ef6c7645cd839 2024-12-05T00:42:20,114 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into e7e4ac0fad0443b1b04ef6c7645cd839(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:20,114 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:20,114 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359340071; duration=0sec 2024-12-05T00:42:20,115 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:20,115 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:20,115 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:20,117 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:20,117 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:20,117 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,117 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3a184bc984134c6a9fe8874f366ae69c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/449abbb6328e488a9e562e792be51d70, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/675da4d58b5e46268d407111adcde812] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=35.7 K 2024-12-05T00:42:20,118 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a184bc984134c6a9fe8874f366ae69c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733359337474 2024-12-05T00:42:20,118 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 449abbb6328e488a9e562e792be51d70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733359338040 2024-12-05T00:42:20,119 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 675da4d58b5e46268d407111adcde812, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733359338661 2024-12-05T00:42:20,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742089_1265 (size=12493) 2024-12-05T00:42:20,132 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:20,132 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/ffd4bb5634644ce29cf26737d2934566 is 50, key is test_row_0/C:col10/1733359338772/Put/seqid=0 2024-12-05T00:42:20,137 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/639430b1ec4e4715a444ed552dd73fd2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/639430b1ec4e4715a444ed552dd73fd2 2024-12-05T00:42:20,149 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 639430b1ec4e4715a444ed552dd73fd2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:20,149 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:20,149 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359340071; duration=0sec 2024-12-05T00:42:20,149 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:20,149 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:20,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742090_1266 (size=12493) 2024-12-05T00:42:20,176 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/ffd4bb5634644ce29cf26737d2934566 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/ffd4bb5634644ce29cf26737d2934566 2024-12-05T00:42:20,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T00:42:20,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:42:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:20,184 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into ffd4bb5634644ce29cf26737d2934566(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:20,184 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:20,184 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359340071; duration=0sec 2024-12-05T00:42:20,184 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:20,184 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/622be16a6bb54e839cd8be3d7bbd8aa7 is 50, key is test_row_0/A:col10/1733359338833/Put/seqid=0 2024-12-05T00:42:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742091_1267 (size=12151) 2024-12-05T00:42:20,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/622be16a6bb54e839cd8be3d7bbd8aa7 2024-12-05T00:42:20,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:20,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f501d8216b2f4c58978de5c223490ab6 is 50, key is test_row_0/B:col10/1733359338833/Put/seqid=0 2024-12-05T00:42:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359400221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359400221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359400222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742092_1268 (size=12151) 2024-12-05T00:42:20,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359400331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359400331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359400331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359400542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359400542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359400542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,663 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f501d8216b2f4c58978de5c223490ab6 2024-12-05T00:42:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/0b19f237c6cb4d879062f91beb53f9b8 is 50, key is test_row_0/C:col10/1733359338833/Put/seqid=0 2024-12-05T00:42:20,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742093_1269 (size=12151) 2024-12-05T00:42:20,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/0b19f237c6cb4d879062f91beb53f9b8 2024-12-05T00:42:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/622be16a6bb54e839cd8be3d7bbd8aa7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/622be16a6bb54e839cd8be3d7bbd8aa7 2024-12-05T00:42:20,729 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/622be16a6bb54e839cd8be3d7bbd8aa7, entries=150, sequenceid=182, filesize=11.9 K 2024-12-05T00:42:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f501d8216b2f4c58978de5c223490ab6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f501d8216b2f4c58978de5c223490ab6 2024-12-05T00:42:20,735 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f501d8216b2f4c58978de5c223490ab6, entries=150, sequenceid=182, filesize=11.9 K 2024-12-05T00:42:20,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/0b19f237c6cb4d879062f91beb53f9b8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/0b19f237c6cb4d879062f91beb53f9b8 2024-12-05T00:42:20,742 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/0b19f237c6cb4d879062f91beb53f9b8, entries=150, sequenceid=182, filesize=11.9 K 2024-12-05T00:42:20,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8052c0b234f27c5e419e64636802bdde in 562ms, sequenceid=182, compaction requested=false 2024-12-05T00:42:20,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:20,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:20,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-05T00:42:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-05T00:42:20,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T00:42:20,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8790 sec 2024-12-05T00:42:20,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.8850 sec 2024-12-05T00:42:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:20,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T00:42:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:20,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/a5a8a301c9a7411e968f8dba6e144879 is 50, key is test_row_0/A:col10/1733359340853/Put/seqid=0 2024-12-05T00:42:20,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742094_1270 (size=16931) 2024-12-05T00:42:20,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/a5a8a301c9a7411e968f8dba6e144879 2024-12-05T00:42:20,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/00e17e92edaa46458d83bf24535bfb87 is 50, key is test_row_0/B:col10/1733359340853/Put/seqid=0 2024-12-05T00:42:20,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359400885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359400890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359400890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742095_1271 (size=12151) 2024-12-05T00:42:20,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359400982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,984 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 8052c0b234f27c5e419e64636802bdde of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:20,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359400992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:20,994 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 8052c0b234f27c5e419e64636802bdde of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:20,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359401001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359401001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359401001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359401211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359401211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359401211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/00e17e92edaa46458d83bf24535bfb87 2024-12-05T00:42:21,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e53fe97bc2d44ef594f8f27d55639a55 is 50, key is test_row_0/C:col10/1733359340853/Put/seqid=0 2024-12-05T00:42:21,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742096_1272 (size=12151) 2024-12-05T00:42:21,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e53fe97bc2d44ef594f8f27d55639a55 2024-12-05T00:42:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/a5a8a301c9a7411e968f8dba6e144879 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a5a8a301c9a7411e968f8dba6e144879 2024-12-05T00:42:21,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a5a8a301c9a7411e968f8dba6e144879, entries=250, sequenceid=198, filesize=16.5 K 2024-12-05T00:42:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/00e17e92edaa46458d83bf24535bfb87 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/00e17e92edaa46458d83bf24535bfb87 2024-12-05T00:42:21,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/00e17e92edaa46458d83bf24535bfb87, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T00:42:21,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e53fe97bc2d44ef594f8f27d55639a55 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e53fe97bc2d44ef594f8f27d55639a55 2024-12-05T00:42:21,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e53fe97bc2d44ef594f8f27d55639a55, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T00:42:21,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8052c0b234f27c5e419e64636802bdde in 489ms, sequenceid=198, compaction requested=true 2024-12-05T00:42:21,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:21,345 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:21,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:21,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:21,348 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:21,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:21,349 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:21,349 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:21,349 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:21,349 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/e7e4ac0fad0443b1b04ef6c7645cd839, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f501d8216b2f4c58978de5c223490ab6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/00e17e92edaa46458d83bf24535bfb87] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=35.9 K 2024-12-05T00:42:21,349 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:21,350 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:21,350 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:21,350 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/639430b1ec4e4715a444ed552dd73fd2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/622be16a6bb54e839cd8be3d7bbd8aa7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a5a8a301c9a7411e968f8dba6e144879] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=40.6 K 2024-12-05T00:42:21,350 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e4ac0fad0443b1b04ef6c7645cd839, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733359338661 2024-12-05T00:42:21,350 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 639430b1ec4e4715a444ed552dd73fd2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733359338661 2024-12-05T00:42:21,350 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting f501d8216b2f4c58978de5c223490ab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733359338808 2024-12-05T00:42:21,351 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622be16a6bb54e839cd8be3d7bbd8aa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733359338808 2024-12-05T00:42:21,351 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 00e17e92edaa46458d83bf24535bfb87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733359340220 2024-12-05T00:42:21,351 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5a8a301c9a7411e968f8dba6e144879, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733359340219 2024-12-05T00:42:21,365 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:21,365 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:21,365 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/04a71f20b04f4f25a28852ce7a412561 is 50, key is test_row_0/A:col10/1733359340853/Put/seqid=0 2024-12-05T00:42:21,366 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/8a8bf08bb0ca46b693afb52393718132 is 50, key is test_row_0/B:col10/1733359340853/Put/seqid=0 2024-12-05T00:42:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742097_1273 (size=12595) 2024-12-05T00:42:21,402 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/8a8bf08bb0ca46b693afb52393718132 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/8a8bf08bb0ca46b693afb52393718132 2024-12-05T00:42:21,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742098_1274 (size=12595) 2024-12-05T00:42:21,416 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into 8a8bf08bb0ca46b693afb52393718132(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:21,416 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:21,416 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359341348; duration=0sec 2024-12-05T00:42:21,416 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:21,416 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:21,416 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:21,417 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:21,417 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:21,418 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:21,418 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/ffd4bb5634644ce29cf26737d2934566, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/0b19f237c6cb4d879062f91beb53f9b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e53fe97bc2d44ef594f8f27d55639a55] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=35.9 K 2024-12-05T00:42:21,419 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ffd4bb5634644ce29cf26737d2934566, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733359338661 2024-12-05T00:42:21,420 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b19f237c6cb4d879062f91beb53f9b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733359338808 2024-12-05T00:42:21,420 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e53fe97bc2d44ef594f8f27d55639a55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733359340220 2024-12-05T00:42:21,432 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:21,432 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/cc47eed445d54984b9443412e27986b3 is 50, key is test_row_0/C:col10/1733359340853/Put/seqid=0 2024-12-05T00:42:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742099_1275 (size=12595) 2024-12-05T00:42:21,483 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/cc47eed445d54984b9443412e27986b3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cc47eed445d54984b9443412e27986b3 2024-12-05T00:42:21,491 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into cc47eed445d54984b9443412e27986b3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:21,491 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:21,491 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359341349; duration=0sec 2024-12-05T00:42:21,492 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:21,492 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:21,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:42:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:21,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:21,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/462c5d97589c4632be55ebaf744a42b6 is 50, key is test_row_0/A:col10/1733359341523/Put/seqid=0 2024-12-05T00:42:21,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359401545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359401550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359401553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742100_1276 (size=14541) 2024-12-05T00:42:21,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/462c5d97589c4632be55ebaf744a42b6 2024-12-05T00:42:21,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c10cd996fb174264be9249f0cacfbd86 is 50, key is test_row_0/B:col10/1733359341523/Put/seqid=0 2024-12-05T00:42:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742101_1277 (size=12151) 2024-12-05T00:42:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359401662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359401662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359401672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,817 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/04a71f20b04f4f25a28852ce7a412561 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04a71f20b04f4f25a28852ce7a412561 2024-12-05T00:42:21,822 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 04a71f20b04f4f25a28852ce7a412561(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:21,822 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:21,822 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359341345; duration=0sec 2024-12-05T00:42:21,822 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:21,822 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:21,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359401871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359401871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:21,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359401882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c10cd996fb174264be9249f0cacfbd86 2024-12-05T00:42:22,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/093230dcd46c47b29eb01f4111a1e44c is 50, key is test_row_0/C:col10/1733359341523/Put/seqid=0 2024-12-05T00:42:22,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742102_1278 (size=12151) 2024-12-05T00:42:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T00:42:22,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:22,032 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-05T00:42:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T00:42:22,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:22,035 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:22,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T00:42:22,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359402181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359402181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T00:42:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:22,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359402192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T00:42:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:22,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T00:42:22,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/093230dcd46c47b29eb01f4111a1e44c 2024-12-05T00:42:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/462c5d97589c4632be55ebaf744a42b6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/462c5d97589c4632be55ebaf744a42b6 2024-12-05T00:42:22,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/462c5d97589c4632be55ebaf744a42b6, entries=200, sequenceid=220, filesize=14.2 K 2024-12-05T00:42:22,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/c10cd996fb174264be9249f0cacfbd86 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c10cd996fb174264be9249f0cacfbd86 2024-12-05T00:42:22,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c10cd996fb174264be9249f0cacfbd86, entries=150, sequenceid=220, filesize=11.9 K 2024-12-05T00:42:22,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/093230dcd46c47b29eb01f4111a1e44c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/093230dcd46c47b29eb01f4111a1e44c 2024-12-05T00:42:22,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/093230dcd46c47b29eb01f4111a1e44c, entries=150, sequenceid=220, filesize=11.9 K 2024-12-05T00:42:22,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8052c0b234f27c5e419e64636802bdde in 930ms, sequenceid=220, compaction requested=false 2024-12-05T00:42:22,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:22,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T00:42:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:22,493 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:42:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/f3ed6a5b0ec4460f8eb2026f57a8277c is 50, key is test_row_0/A:col10/1733359341548/Put/seqid=0 2024-12-05T00:42:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742103_1279 (size=12151) 2024-12-05T00:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T00:42:22,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359402723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359402722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359402724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359402831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:22,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/f3ed6a5b0ec4460f8eb2026f57a8277c 2024-12-05T00:42:22,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/774b5e6450c44dbd8e70ce1af2958aff is 50, key is test_row_0/B:col10/1733359341548/Put/seqid=0 2024-12-05T00:42:22,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742104_1280 (size=12151) 2024-12-05T00:42:22,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/774b5e6450c44dbd8e70ce1af2958aff 2024-12-05T00:42:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/b1b9fea145c246018229fc29216bb291 is 50, key is test_row_0/C:col10/1733359341548/Put/seqid=0 2024-12-05T00:42:22,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742105_1281 (size=12151) 2024-12-05T00:42:23,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359403002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,004 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 8052c0b234f27c5e419e64636802bdde of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359403012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,014 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 8052c0b234f27c5e419e64636802bdde of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359403041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359403041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359403041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T00:42:23,329 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/b1b9fea145c246018229fc29216bb291 2024-12-05T00:42:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/f3ed6a5b0ec4460f8eb2026f57a8277c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/f3ed6a5b0ec4460f8eb2026f57a8277c 2024-12-05T00:42:23,338 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/f3ed6a5b0ec4460f8eb2026f57a8277c, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T00:42:23,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/774b5e6450c44dbd8e70ce1af2958aff as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/774b5e6450c44dbd8e70ce1af2958aff 2024-12-05T00:42:23,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/774b5e6450c44dbd8e70ce1af2958aff, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T00:42:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/b1b9fea145c246018229fc29216bb291 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/b1b9fea145c246018229fc29216bb291 2024-12-05T00:42:23,350 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/b1b9fea145c246018229fc29216bb291, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T00:42:23,352 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8052c0b234f27c5e419e64636802bdde in 859ms, sequenceid=233, compaction requested=true 2024-12-05T00:42:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-05T00:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-05T00:42:23,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359342704, max timestamp=1733359343355Num uniques 859; 2024-12-05T00:42:23,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359342704, max timestamp=1733359343355Num uniques 859; 2024-12-05T00:42:23,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:23,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:23,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359342704, max timestamp=1733359343355Num uniques 859; 2024-12-05T00:42:23,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:23,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-05T00:42:23,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-12-05T00:42:23,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3320 sec 2024-12-05T00:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:23,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-05T00:42:23,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:23,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/88b136a058b747fab387c94cad68a947 is 50, key is test_row_0/A:col10/1733359343365/Put/seqid=0 2024-12-05T00:42:23,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742106_1282 (size=12301) 2024-12-05T00:42:23,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/88b136a058b747fab387c94cad68a947 2024-12-05T00:42:23,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/af2eb688f90b463bb8f0f36367ea323c is 50, key is test_row_0/B:col10/1733359343365/Put/seqid=0 2024-12-05T00:42:23,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742107_1283 (size=12301) 2024-12-05T00:42:23,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/af2eb688f90b463bb8f0f36367ea323c 2024-12-05T00:42:23,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359343423, max timestamp=1733359343470Num uniques 859; 2024-12-05T00:42:23,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359343423, max timestamp=1733359343470Num uniques 859; 2024-12-05T00:42:23,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:23,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:23,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359343423, max timestamp=1733359343470Num uniques 859; 2024-12-05T00:42:23,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:23,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/fe9e76d9756343318455af1eba4243e3 is 50, key is test_row_0/C:col10/1733359343365/Put/seqid=0 2024-12-05T00:42:23,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742108_1284 (size=12301) 2024-12-05T00:42:23,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/fe9e76d9756343318455af1eba4243e3 2024-12-05T00:42:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/88b136a058b747fab387c94cad68a947 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/88b136a058b747fab387c94cad68a947 2024-12-05T00:42:23,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/88b136a058b747fab387c94cad68a947, entries=150, sequenceid=259, filesize=12.0 K 2024-12-05T00:42:23,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/af2eb688f90b463bb8f0f36367ea323c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/af2eb688f90b463bb8f0f36367ea323c 2024-12-05T00:42:23,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/af2eb688f90b463bb8f0f36367ea323c, entries=150, sequenceid=259, filesize=12.0 K 2024-12-05T00:42:23,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/fe9e76d9756343318455af1eba4243e3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/fe9e76d9756343318455af1eba4243e3 2024-12-05T00:42:23,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359403534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/fe9e76d9756343318455af1eba4243e3, entries=150, sequenceid=259, filesize=12.0 K 2024-12-05T00:42:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359403535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=134.18 KB/137400 for 8052c0b234f27c5e419e64636802bdde in 177ms, sequenceid=259, compaction requested=true 2024-12-05T00:42:23,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:23,543 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:23,544 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733359343505, max timestamp=1733359343543Num uniques 1018; 2024-12-05T00:42:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733359343505, max timestamp=1733359343543Num uniques 1018; 2024-12-05T00:42:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-05T00:42:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-05T00:42:23,547 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51588 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:23,548 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:23,548 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:23,548 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04a71f20b04f4f25a28852ce7a412561, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/462c5d97589c4632be55ebaf744a42b6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/f3ed6a5b0ec4460f8eb2026f57a8277c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/88b136a058b747fab387c94cad68a947] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=50.4 K 2024-12-05T00:42:23,548 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:23,548 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a71f20b04f4f25a28852ce7a412561, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733359340220 2024-12-05T00:42:23,548 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:23,548 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:23,548 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/8a8bf08bb0ca46b693afb52393718132, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c10cd996fb174264be9249f0cacfbd86, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/774b5e6450c44dbd8e70ce1af2958aff, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/af2eb688f90b463bb8f0f36367ea323c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=48.0 K 2024-12-05T00:42:23,549 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462c5d97589c4632be55ebaf744a42b6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733359340874 2024-12-05T00:42:23,549 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8bf08bb0ca46b693afb52393718132, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733359340220 2024-12-05T00:42:23,549 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ed6a5b0ec4460f8eb2026f57a8277c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733359341543 2024-12-05T00:42:23,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733359343505, max timestamp=1733359343543Num uniques 1018; 2024-12-05T00:42:23,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T00:42:23,549 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b136a058b747fab387c94cad68a947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733359343364 2024-12-05T00:42:23,549 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c10cd996fb174264be9249f0cacfbd86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733359340874 2024-12-05T00:42:23,550 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 774b5e6450c44dbd8e70ce1af2958aff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733359341543 2024-12-05T00:42:23,551 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting af2eb688f90b463bb8f0f36367ea323c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733359343364 2024-12-05T00:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:23,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=55.91 KB heapSize=145.47 KB 2024-12-05T00:42:23,571 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:23,571 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/973429f57e104741b7e3d09d98d61a27 is 50, key is test_row_0/B:col10/1733359343365/Put/seqid=0 2024-12-05T00:42:23,582 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:23,583 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/e6d7598766804f1c8cf69703f55201cf is 50, key is test_row_0/A:col10/1733359343365/Put/seqid=0 2024-12-05T00:42:23,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/e76e2f4421e04c1980574397504732c5 is 50, key is test_row_0/A:col10/1733359343505/Put/seqid=0 2024-12-05T00:42:23,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742109_1285 (size=12881) 2024-12-05T00:42:23,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742110_1286 (size=12301) 2024-12-05T00:42:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742111_1287 (size=12881) 2024-12-05T00:42:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359343628, max timestamp=1733359343655Num uniques 859; 2024-12-05T00:42:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359343628, max timestamp=1733359343655Num uniques 859; 2024-12-05T00:42:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359343628, max timestamp=1733359343655Num uniques 859; 2024-12-05T00:42:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359403676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359403677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359403679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359403791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359403791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359403791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359404002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359404002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,007 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/973429f57e104741b7e3d09d98d61a27 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/973429f57e104741b7e3d09d98d61a27 2024-12-05T00:42:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359404002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,014 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into 973429f57e104741b7e3d09d98d61a27(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:24,014 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:24,014 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=12, startTime=1733359343544; duration=0sec 2024-12-05T00:42:24,014 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:24,014 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:24,014 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:24,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/e76e2f4421e04c1980574397504732c5 2024-12-05T00:42:24,020 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:24,020 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:24,020 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:24,020 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cc47eed445d54984b9443412e27986b3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/093230dcd46c47b29eb01f4111a1e44c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/b1b9fea145c246018229fc29216bb291, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/fe9e76d9756343318455af1eba4243e3] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=48.0 K 2024-12-05T00:42:24,020 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting cc47eed445d54984b9443412e27986b3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733359340220 2024-12-05T00:42:24,021 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 093230dcd46c47b29eb01f4111a1e44c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733359340874 2024-12-05T00:42:24,021 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b9fea145c246018229fc29216bb291, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733359341543 2024-12-05T00:42:24,022 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9e76d9756343318455af1eba4243e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733359343364 2024-12-05T00:42:24,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/1dbfbf39cd1e4b57959f9032642786bf is 50, key is test_row_0/B:col10/1733359343505/Put/seqid=0 2024-12-05T00:42:24,030 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/e6d7598766804f1c8cf69703f55201cf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e6d7598766804f1c8cf69703f55201cf 2024-12-05T00:42:24,041 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into e6d7598766804f1c8cf69703f55201cf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:24,041 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:24,041 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=12, startTime=1733359343543; duration=0sec 2024-12-05T00:42:24,041 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:24,041 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:24,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742112_1288 (size=12301) 2024-12-05T00:42:24,053 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:24,053 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/5e7024f2042d458a85eacdbaeb520141 is 50, key is test_row_0/C:col10/1733359343365/Put/seqid=0 2024-12-05T00:42:24,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742113_1289 (size=12881) 2024-12-05T00:42:24,099 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/5e7024f2042d458a85eacdbaeb520141 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5e7024f2042d458a85eacdbaeb520141 2024-12-05T00:42:24,103 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into 5e7024f2042d458a85eacdbaeb520141(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:24,103 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:24,103 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=12, startTime=1733359343544; duration=0sec 2024-12-05T00:42:24,103 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:24,103 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T00:42:24,181 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:24,182 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-05T00:42:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T00:42:24,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:24,184 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:24,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T00:42:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359404311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359404312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359404312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T00:42:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:24,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:24,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/1dbfbf39cd1e4b57959f9032642786bf 2024-12-05T00:42:24,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/2e9b6ca928f24ba3b5cde857dd0308aa is 50, key is test_row_0/C:col10/1733359343505/Put/seqid=0 2024-12-05T00:42:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742114_1290 (size=12301) 2024-12-05T00:42:24,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/2e9b6ca928f24ba3b5cde857dd0308aa 2024-12-05T00:42:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/e76e2f4421e04c1980574397504732c5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e76e2f4421e04c1980574397504732c5 2024-12-05T00:42:24,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e76e2f4421e04c1980574397504732c5, entries=150, sequenceid=300, filesize=12.0 K 2024-12-05T00:42:24,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/1dbfbf39cd1e4b57959f9032642786bf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/1dbfbf39cd1e4b57959f9032642786bf 2024-12-05T00:42:24,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T00:42:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:24,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:24,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/1dbfbf39cd1e4b57959f9032642786bf, entries=150, sequenceid=300, filesize=12.0 K 2024-12-05T00:42:24,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/2e9b6ca928f24ba3b5cde857dd0308aa as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/2e9b6ca928f24ba3b5cde857dd0308aa 2024-12-05T00:42:24,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/2e9b6ca928f24ba3b5cde857dd0308aa, entries=150, sequenceid=300, filesize=12.0 K 2024-12-05T00:42:24,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~55.91 KB/57250, heapSize ~145.42 KB/148912, currentSize=134.18 KB/137400 for 8052c0b234f27c5e419e64636802bdde in 930ms, sequenceid=300, compaction requested=false 2024-12-05T00:42:24,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T00:42:24,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T00:42:24,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:24,643 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-05T00:42:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:24,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/21189ce66ef0476fb2953288881def61 is 50, key is test_row_0/A:col10/1733359343676/Put/seqid=0 2024-12-05T00:42:24,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742115_1291 (size=12301) 2024-12-05T00:42:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T00:42:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:24,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:24,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359404835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359404835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359404836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359404952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359404952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359404952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,051 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/21189ce66ef0476fb2953288881def61 2024-12-05T00:42:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/27603056f58c40ba96504e6864023fed is 50, key is test_row_0/B:col10/1733359343676/Put/seqid=0 2024-12-05T00:42:25,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742116_1292 (size=12301) 2024-12-05T00:42:25,084 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/27603056f58c40ba96504e6864023fed 2024-12-05T00:42:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/7f2d75c2d6cc4407855518269756d19c is 50, key is test_row_0/C:col10/1733359343676/Put/seqid=0 2024-12-05T00:42:25,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742117_1293 (size=12301) 2024-12-05T00:42:25,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359405162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:25,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359405162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:25,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359405162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T00:42:25,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359405471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:25,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359405471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359405475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:25,514 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/7f2d75c2d6cc4407855518269756d19c 2024-12-05T00:42:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/21189ce66ef0476fb2953288881def61 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/21189ce66ef0476fb2953288881def61 2024-12-05T00:42:25,523 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/21189ce66ef0476fb2953288881def61, entries=150, sequenceid=338, filesize=12.0 K 2024-12-05T00:42:25,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/27603056f58c40ba96504e6864023fed as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/27603056f58c40ba96504e6864023fed 2024-12-05T00:42:25,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/27603056f58c40ba96504e6864023fed, entries=150, sequenceid=338, filesize=12.0 K 2024-12-05T00:42:25,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/7f2d75c2d6cc4407855518269756d19c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7f2d75c2d6cc4407855518269756d19c 2024-12-05T00:42:25,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7f2d75c2d6cc4407855518269756d19c, entries=150, sequenceid=338, filesize=12.0 K 2024-12-05T00:42:25,547 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 8052c0b234f27c5e419e64636802bdde in 905ms, sequenceid=338, compaction requested=true 2024-12-05T00:42:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-05T00:42:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-05T00:42:25,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-05T00:42:25,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3650 sec 2024-12-05T00:42:25,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3720 sec 2024-12-05T00:42:25,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:42:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:25,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/2a1cc77206ed49cc8791dd9649b51b13 is 50, key is test_row_0/A:col10/1733359345983/Put/seqid=0 2024-12-05T00:42:26,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742118_1294 (size=14741) 2024-12-05T00:42:26,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/2a1cc77206ed49cc8791dd9649b51b13 2024-12-05T00:42:26,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/94b5e09f9364441f876c70f2fb7ae8ac is 50, key is test_row_0/B:col10/1733359345983/Put/seqid=0 2024-12-05T00:42:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359406029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359406030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359406030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742119_1295 (size=12301) 2024-12-05T00:42:26,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359406142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359406142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359406142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T00:42:26,342 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:26,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-05T00:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T00:42:26,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:26,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:26,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:26,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359406352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359406352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359406352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T00:42:26,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/94b5e09f9364441f876c70f2fb7ae8ac 2024-12-05T00:42:26,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/564d01fa0ecf4358986638fcdb685a4d is 50, key is test_row_0/C:col10/1733359345983/Put/seqid=0 2024-12-05T00:42:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742120_1296 (size=12301) 2024-12-05T00:42:26,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T00:42:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T00:42:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T00:42:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359406662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359406662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359406662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:26,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T00:42:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:26,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/564d01fa0ecf4358986638fcdb685a4d 2024-12-05T00:42:26,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/2a1cc77206ed49cc8791dd9649b51b13 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2a1cc77206ed49cc8791dd9649b51b13 2024-12-05T00:42:26,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2a1cc77206ed49cc8791dd9649b51b13, entries=200, sequenceid=351, filesize=14.4 K 2024-12-05T00:42:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/94b5e09f9364441f876c70f2fb7ae8ac as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/94b5e09f9364441f876c70f2fb7ae8ac 2024-12-05T00:42:26,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/94b5e09f9364441f876c70f2fb7ae8ac, entries=150, sequenceid=351, filesize=12.0 K 2024-12-05T00:42:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/564d01fa0ecf4358986638fcdb685a4d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/564d01fa0ecf4358986638fcdb685a4d 2024-12-05T00:42:26,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/564d01fa0ecf4358986638fcdb685a4d, entries=150, sequenceid=351, filesize=12.0 K 2024-12-05T00:42:26,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8052c0b234f27c5e419e64636802bdde in 908ms, sequenceid=351, compaction requested=true 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:26,892 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T00:42:26,893 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:26,894 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:26,894 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:26,894 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:26,894 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:26,894 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,894 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,894 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e6d7598766804f1c8cf69703f55201cf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e76e2f4421e04c1980574397504732c5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/21189ce66ef0476fb2953288881def61, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2a1cc77206ed49cc8791dd9649b51b13] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=51 K 2024-12-05T00:42:26,894 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/973429f57e104741b7e3d09d98d61a27, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/1dbfbf39cd1e4b57959f9032642786bf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/27603056f58c40ba96504e6864023fed, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/94b5e09f9364441f876c70f2fb7ae8ac] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=48.6 K 2024-12-05T00:42:26,895 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 973429f57e104741b7e3d09d98d61a27, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733359343364 2024-12-05T00:42:26,895 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d7598766804f1c8cf69703f55201cf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733359343364 2024-12-05T00:42:26,895 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e76e2f4421e04c1980574397504732c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733359343505 2024-12-05T00:42:26,895 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dbfbf39cd1e4b57959f9032642786bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733359343505 2024-12-05T00:42:26,895 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 21189ce66ef0476fb2953288881def61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733359343675 2024-12-05T00:42:26,895 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27603056f58c40ba96504e6864023fed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733359343675 2024-12-05T00:42:26,896 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1cc77206ed49cc8791dd9649b51b13, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733359344828 2024-12-05T00:42:26,896 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b5e09f9364441f876c70f2fb7ae8ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733359344828 2024-12-05T00:42:26,912 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#253 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:26,912 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#252 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:26,913 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/8871e3388c374e8996cff8f3eb5c22a7 is 50, key is test_row_0/A:col10/1733359345983/Put/seqid=0 2024-12-05T00:42:26,913 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/eba7d93972154c04a75bc3e78866b89d is 50, key is test_row_0/B:col10/1733359345983/Put/seqid=0 2024-12-05T00:42:26,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742122_1298 (size=13017) 2024-12-05T00:42:26,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-05T00:42:26,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:26,956 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:42:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/abd0dfaf7439417088c12b09780bcb05 is 50, key is test_row_0/A:col10/1733359346019/Put/seqid=0 2024-12-05T00:42:26,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742121_1297 (size=13017) 2024-12-05T00:42:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T00:42:26,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742123_1299 (size=12301) 2024-12-05T00:42:26,978 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/abd0dfaf7439417088c12b09780bcb05 2024-12-05T00:42:26,994 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/eba7d93972154c04a75bc3e78866b89d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/eba7d93972154c04a75bc3e78866b89d 2024-12-05T00:42:27,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/87f0c121c3384a6db0f774a8690a0d6d is 50, key is test_row_0/B:col10/1733359346019/Put/seqid=0 2024-12-05T00:42:27,015 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into eba7d93972154c04a75bc3e78866b89d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:27,015 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:27,016 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=12, startTime=1733359346892; duration=0sec 2024-12-05T00:42:27,016 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:27,016 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:27,016 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:42:27,019 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:42:27,019 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:27,019 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,019 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5e7024f2042d458a85eacdbaeb520141, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/2e9b6ca928f24ba3b5cde857dd0308aa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7f2d75c2d6cc4407855518269756d19c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/564d01fa0ecf4358986638fcdb685a4d] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=48.6 K 2024-12-05T00:42:27,020 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e7024f2042d458a85eacdbaeb520141, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733359343364 2024-12-05T00:42:27,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e9b6ca928f24ba3b5cde857dd0308aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733359343505 2024-12-05T00:42:27,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2d75c2d6cc4407855518269756d19c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733359343675 2024-12-05T00:42:27,021 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 564d01fa0ecf4358986638fcdb685a4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733359344828 2024-12-05T00:42:27,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742124_1300 (size=12301) 2024-12-05T00:42:27,026 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/87f0c121c3384a6db0f774a8690a0d6d 2024-12-05T00:42:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/5fc9f929b096426294e52490471d9b61 is 50, key is test_row_0/C:col10/1733359346019/Put/seqid=0 2024-12-05T00:42:27,062 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:27,063 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/bb2f8ec28195486ea3f72f338c11cad1 is 50, key is test_row_0/C:col10/1733359345983/Put/seqid=0 2024-12-05T00:42:27,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359407058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359407060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742125_1301 (size=12301) 2024-12-05T00:42:27,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/5fc9f929b096426294e52490471d9b61 2024-12-05T00:42:27,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/abd0dfaf7439417088c12b09780bcb05 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/abd0dfaf7439417088c12b09780bcb05 2024-12-05T00:42:27,101 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/abd0dfaf7439417088c12b09780bcb05, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T00:42:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/87f0c121c3384a6db0f774a8690a0d6d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/87f0c121c3384a6db0f774a8690a0d6d 2024-12-05T00:42:27,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742126_1302 (size=13017) 2024-12-05T00:42:27,107 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/87f0c121c3384a6db0f774a8690a0d6d, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T00:42:27,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/5fc9f929b096426294e52490471d9b61 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5fc9f929b096426294e52490471d9b61 2024-12-05T00:42:27,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5fc9f929b096426294e52490471d9b61, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T00:42:27,115 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8052c0b234f27c5e419e64636802bdde in 158ms, sequenceid=371, compaction requested=false 2024-12-05T00:42:27,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:27,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-05T00:42:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-05T00:42:27,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-05T00:42:27,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 773 msec 2024-12-05T00:42:27,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 778 msec 2024-12-05T00:42:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:27,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:42:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:27,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/2771508870fb4925bfc02fa7d8643212 is 50, key is test_row_0/A:col10/1733359347175/Put/seqid=0 2024-12-05T00:42:27,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742127_1303 (size=12301) 2024-12-05T00:42:27,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359407198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359407199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359407201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359407208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359407210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359407321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359407321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359407322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359407322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359407322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,369 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/8871e3388c374e8996cff8f3eb5c22a7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8871e3388c374e8996cff8f3eb5c22a7 2024-12-05T00:42:27,378 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 8871e3388c374e8996cff8f3eb5c22a7(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:27,378 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:27,378 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=12, startTime=1733359346892; duration=0sec 2024-12-05T00:42:27,378 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:27,378 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T00:42:27,482 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:27,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-05T00:42:27,486 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T00:42:27,488 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:27,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:27,510 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/bb2f8ec28195486ea3f72f338c11cad1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/bb2f8ec28195486ea3f72f338c11cad1 2024-12-05T00:42:27,519 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into bb2f8ec28195486ea3f72f338c11cad1(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:27,519 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:27,519 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=12, startTime=1733359346892; duration=0sec 2024-12-05T00:42:27,519 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:27,519 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359407531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359407531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359407531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359407532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359407532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T00:42:27,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/2771508870fb4925bfc02fa7d8643212 2024-12-05T00:42:27,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0abf18449c9046768c28a87e55c157e0 is 50, key is test_row_0/B:col10/1733359347175/Put/seqid=0 2024-12-05T00:42:27,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742128_1304 (size=12301) 2024-12-05T00:42:27,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:27,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:27,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:27,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T00:42:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359407841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359407841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359407841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359407842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359407842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:27,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:27,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:27,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:27,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0abf18449c9046768c28a87e55c157e0 2024-12-05T00:42:28,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/39d75a46f0134c7283d75beeda79c3de is 50, key is test_row_0/C:col10/1733359347175/Put/seqid=0 2024-12-05T00:42:28,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742129_1305 (size=12301) 2024-12-05T00:42:28,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T00:42:28,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359408352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359408352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359408352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359408352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359408353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:28,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:28,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/39d75a46f0134c7283d75beeda79c3de 2024-12-05T00:42:28,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/2771508870fb4925bfc02fa7d8643212 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2771508870fb4925bfc02fa7d8643212 2024-12-05T00:42:28,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2771508870fb4925bfc02fa7d8643212, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T00:42:28,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0abf18449c9046768c28a87e55c157e0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0abf18449c9046768c28a87e55c157e0 2024-12-05T00:42:28,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0abf18449c9046768c28a87e55c157e0, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T00:42:28,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/39d75a46f0134c7283d75beeda79c3de as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/39d75a46f0134c7283d75beeda79c3de 2024-12-05T00:42:28,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/39d75a46f0134c7283d75beeda79c3de, entries=150, sequenceid=387, filesize=12.0 K 2024-12-05T00:42:28,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8052c0b234f27c5e419e64636802bdde in 1337ms, sequenceid=387, compaction requested=true 2024-12-05T00:42:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:28,514 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:28,514 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:28,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:28,515 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:28,515 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:28,515 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:28,515 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:28,515 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,515 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,515 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8871e3388c374e8996cff8f3eb5c22a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/abd0dfaf7439417088c12b09780bcb05, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2771508870fb4925bfc02fa7d8643212] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=36.7 K 2024-12-05T00:42:28,515 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/eba7d93972154c04a75bc3e78866b89d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/87f0c121c3384a6db0f774a8690a0d6d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0abf18449c9046768c28a87e55c157e0] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=36.7 K 2024-12-05T00:42:28,516 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8871e3388c374e8996cff8f3eb5c22a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733359344828 2024-12-05T00:42:28,516 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting eba7d93972154c04a75bc3e78866b89d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733359344828 2024-12-05T00:42:28,516 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f0c121c3384a6db0f774a8690a0d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733359346012 2024-12-05T00:42:28,516 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd0dfaf7439417088c12b09780bcb05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733359346012 2024-12-05T00:42:28,516 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0abf18449c9046768c28a87e55c157e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733359347173 2024-12-05T00:42:28,516 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2771508870fb4925bfc02fa7d8643212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733359347173 2024-12-05T00:42:28,525 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:28,525 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9dbbb32cc41543be9e841b21733d50b1 is 50, key is test_row_0/B:col10/1733359347175/Put/seqid=0 2024-12-05T00:42:28,550 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:28,550 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/5ce8700cf6324c4eaa3dc93dacccc52f is 50, key is test_row_0/A:col10/1733359347175/Put/seqid=0 2024-12-05T00:42:28,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742130_1306 (size=13119) 2024-12-05T00:42:28,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-05T00:42:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,576 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T00:42:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:28,580 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9dbbb32cc41543be9e841b21733d50b1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9dbbb32cc41543be9e841b21733d50b1 2024-12-05T00:42:28,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/3f2839085e4f49d39d73e20f4d20280d is 50, key is test_row_0/A:col10/1733359347206/Put/seqid=0 2024-12-05T00:42:28,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742131_1307 (size=13119) 2024-12-05T00:42:28,590 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into 9dbbb32cc41543be9e841b21733d50b1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:28,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:28,590 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359348514; duration=0sec 2024-12-05T00:42:28,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:28,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:28,590 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:28,594 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:28,594 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:28,594 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:28,594 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/bb2f8ec28195486ea3f72f338c11cad1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5fc9f929b096426294e52490471d9b61, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/39d75a46f0134c7283d75beeda79c3de] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=36.7 K 2024-12-05T00:42:28,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742132_1308 (size=12301) 2024-12-05T00:42:28,595 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting bb2f8ec28195486ea3f72f338c11cad1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733359344828 2024-12-05T00:42:28,597 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc9f929b096426294e52490471d9b61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733359346012 2024-12-05T00:42:28,599 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d75a46f0134c7283d75beeda79c3de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733359347173 2024-12-05T00:42:28,600 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/5ce8700cf6324c4eaa3dc93dacccc52f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/5ce8700cf6324c4eaa3dc93dacccc52f 2024-12-05T00:42:28,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/3f2839085e4f49d39d73e20f4d20280d 2024-12-05T00:42:28,613 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 5ce8700cf6324c4eaa3dc93dacccc52f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:28,613 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:28,613 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359348514; duration=0sec 2024-12-05T00:42:28,613 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:28,613 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:28,618 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:28,618 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/51c7921e6f5b4cc5a1a3904c888a3508 is 50, key is test_row_0/C:col10/1733359347175/Put/seqid=0 2024-12-05T00:42:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T00:42:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/cf7f4baddca64bca872afd58c702a76e is 50, key is test_row_0/B:col10/1733359347206/Put/seqid=0 2024-12-05T00:42:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742133_1309 (size=13119) 2024-12-05T00:42:28,680 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/51c7921e6f5b4cc5a1a3904c888a3508 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/51c7921e6f5b4cc5a1a3904c888a3508 2024-12-05T00:42:28,687 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into 51c7921e6f5b4cc5a1a3904c888a3508(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:28,687 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:28,688 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359348514; duration=0sec 2024-12-05T00:42:28,688 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:28,688 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:28,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742134_1310 (size=12301) 2024-12-05T00:42:28,693 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/cf7f4baddca64bca872afd58c702a76e 2024-12-05T00:42:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/a09b76e672e84e80bc07e0e432128bf0 is 50, key is test_row_0/C:col10/1733359347206/Put/seqid=0 2024-12-05T00:42:28,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742135_1311 (size=12301) 2024-12-05T00:42:29,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/a09b76e672e84e80bc07e0e432128bf0 2024-12-05T00:42:29,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/3f2839085e4f49d39d73e20f4d20280d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/3f2839085e4f49d39d73e20f4d20280d 2024-12-05T00:42:29,159 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/3f2839085e4f49d39d73e20f4d20280d, entries=150, sequenceid=406, filesize=12.0 K 2024-12-05T00:42:29,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/cf7f4baddca64bca872afd58c702a76e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/cf7f4baddca64bca872afd58c702a76e 2024-12-05T00:42:29,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/cf7f4baddca64bca872afd58c702a76e, entries=150, sequenceid=406, filesize=12.0 K 2024-12-05T00:42:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/a09b76e672e84e80bc07e0e432128bf0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/a09b76e672e84e80bc07e0e432128bf0 2024-12-05T00:42:29,179 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/a09b76e672e84e80bc07e0e432128bf0, entries=150, sequenceid=406, filesize=12.0 K 2024-12-05T00:42:29,180 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 8052c0b234f27c5e419e64636802bdde in 604ms, sequenceid=406, compaction requested=false 2024-12-05T00:42:29,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:29,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:29,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-05T00:42:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-05T00:42:29,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-05T00:42:29,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6960 sec 2024-12-05T00:42:29,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7040 sec 2024-12-05T00:42:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:29,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:42:29,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:29,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:29,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:29,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:29,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:29,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:29,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/8bdb68c6c9664e3986ad4a21f6937cdf is 50, key is test_row_1/A:col10/1733359349373/Put/seqid=0 2024-12-05T00:42:29,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359349401, max timestamp=1733359349420Num uniques 859; 2024-12-05T00:42:29,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:29,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359349401, max timestamp=1733359349420Num uniques 859; 2024-12-05T00:42:29,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359349401, max timestamp=1733359349420Num uniques 859; 2024-12-05T00:42:29,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:29,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742136_1312 (size=9857) 2024-12-05T00:42:29,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/8bdb68c6c9664e3986ad4a21f6937cdf 2024-12-05T00:42:29,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0663279c3e384927a70bba11a616285e is 50, key is test_row_1/B:col10/1733359349373/Put/seqid=0 2024-12-05T00:42:29,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359409488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359409493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359409493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359409495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742137_1313 (size=9857) 2024-12-05T00:42:29,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359409501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359409612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359409613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359409613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359409613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359409613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-05T00:42:29,631 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:29,632 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-05T00:42:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T00:42:29,634 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:29,635 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:29,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T00:42:29,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T00:42:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:29,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:29,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359409831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359409831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359409831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359409832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359409832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:29,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0663279c3e384927a70bba11a616285e 2024-12-05T00:42:29,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/7b562aae7e514622b6f7334bd2e28bd1 is 50, key is test_row_1/C:col10/1733359349373/Put/seqid=0 2024-12-05T00:42:29,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T00:42:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:29,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T00:42:29,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742138_1314 (size=9857) 2024-12-05T00:42:30,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T00:42:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359410142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359410142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359410142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359410143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359410144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T00:42:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T00:42:30,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/7b562aae7e514622b6f7334bd2e28bd1 2024-12-05T00:42:30,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/8bdb68c6c9664e3986ad4a21f6937cdf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8bdb68c6c9664e3986ad4a21f6937cdf 2024-12-05T00:42:30,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8bdb68c6c9664e3986ad4a21f6937cdf, entries=100, sequenceid=419, filesize=9.6 K 2024-12-05T00:42:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/0663279c3e384927a70bba11a616285e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0663279c3e384927a70bba11a616285e 2024-12-05T00:42:30,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0663279c3e384927a70bba11a616285e, entries=100, sequenceid=419, filesize=9.6 K 2024-12-05T00:42:30,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/7b562aae7e514622b6f7334bd2e28bd1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7b562aae7e514622b6f7334bd2e28bd1 2024-12-05T00:42:30,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7b562aae7e514622b6f7334bd2e28bd1, entries=100, sequenceid=419, filesize=9.6 K 2024-12-05T00:42:30,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8052c0b234f27c5e419e64636802bdde in 1025ms, sequenceid=419, compaction requested=true 2024-12-05T00:42:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:30,400 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:30,400 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:30,403 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:30,403 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:30,403 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,403 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9dbbb32cc41543be9e841b21733d50b1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/cf7f4baddca64bca872afd58c702a76e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0663279c3e384927a70bba11a616285e] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=34.5 K 2024-12-05T00:42:30,403 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:30,403 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:30,403 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,403 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/5ce8700cf6324c4eaa3dc93dacccc52f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/3f2839085e4f49d39d73e20f4d20280d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8bdb68c6c9664e3986ad4a21f6937cdf] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=34.5 K 2024-12-05T00:42:30,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-05T00:42:30,404 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce8700cf6324c4eaa3dc93dacccc52f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733359347173 2024-12-05T00:42:30,404 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dbbb32cc41543be9e841b21733d50b1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733359347173 2024-12-05T00:42:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,404 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T00:42:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:30,405 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f2839085e4f49d39d73e20f4d20280d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733359347194 2024-12-05T00:42:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:30,406 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting cf7f4baddca64bca872afd58c702a76e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733359347194 2024-12-05T00:42:30,407 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bdb68c6c9664e3986ad4a21f6937cdf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733359349368 2024-12-05T00:42:30,407 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0663279c3e384927a70bba11a616285e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733359349368 2024-12-05T00:42:30,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:30,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:30,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:30,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:30,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/770f9064579d4dafad37109a368efbf2 is 50, key is test_row_0/A:col10/1733359349460/Put/seqid=0 2024-12-05T00:42:30,446 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:30,446 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/592b50c55aa34635b4343d272b7afcbc is 50, key is test_row_0/B:col10/1733359347206/Put/seqid=0 2024-12-05T00:42:30,463 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:30,464 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/575033a3bbf14dde962fcde599ad4fe1 is 50, key is test_row_0/A:col10/1733359347206/Put/seqid=0 2024-12-05T00:42:30,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742139_1315 (size=12301) 2024-12-05T00:42:30,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742140_1316 (size=13221) 2024-12-05T00:42:30,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/592b50c55aa34635b4343d272b7afcbc as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/592b50c55aa34635b4343d272b7afcbc 2024-12-05T00:42:30,529 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into 592b50c55aa34635b4343d272b7afcbc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:30,530 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:30,530 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359350400; duration=0sec 2024-12-05T00:42:30,530 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:30,530 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:30,530 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:30,532 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:30,532 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:30,532 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:30,533 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/51c7921e6f5b4cc5a1a3904c888a3508, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/a09b76e672e84e80bc07e0e432128bf0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7b562aae7e514622b6f7334bd2e28bd1] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=34.5 K 2024-12-05T00:42:30,533 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c7921e6f5b4cc5a1a3904c888a3508, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733359347173 2024-12-05T00:42:30,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742141_1317 (size=13221) 2024-12-05T00:42:30,534 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a09b76e672e84e80bc07e0e432128bf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733359347194 2024-12-05T00:42:30,535 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b562aae7e514622b6f7334bd2e28bd1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733359349368 2024-12-05T00:42:30,542 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/575033a3bbf14dde962fcde599ad4fe1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/575033a3bbf14dde962fcde599ad4fe1 2024-12-05T00:42:30,547 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:30,547 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/1660f882050f4d188ce3e60ea0cb56e7 is 50, key is test_row_0/C:col10/1733359347206/Put/seqid=0 2024-12-05T00:42:30,551 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 575033a3bbf14dde962fcde599ad4fe1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:30,551 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:30,551 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359350400; duration=0sec 2024-12-05T00:42:30,551 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:30,552 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742142_1318 (size=13221) 2024-12-05T00:42:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:30,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:30,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359410661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359410662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359410662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359410663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359410669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T00:42:30,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359410771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359410772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359410782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,894 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/770f9064579d4dafad37109a368efbf2 2024-12-05T00:42:30,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/a64407b308234d7894d347291fdf1097 is 50, key is test_row_0/B:col10/1733359349460/Put/seqid=0 2024-12-05T00:42:30,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742143_1319 (size=12301) 2024-12-05T00:42:30,924 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/a64407b308234d7894d347291fdf1097 2024-12-05T00:42:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e255bd125780492ba842b857eca0521e is 50, key is test_row_0/C:col10/1733359349460/Put/seqid=0 2024-12-05T00:42:30,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742144_1320 (size=12301) 2024-12-05T00:42:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359410982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359410982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:30,995 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/1660f882050f4d188ce3e60ea0cb56e7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1660f882050f4d188ce3e60ea0cb56e7 2024-12-05T00:42:30,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359410993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:30,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:30,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,001 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into 1660f882050f4d188ce3e60ea0cb56e7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:31,001 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:31,001 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359350407; duration=0sec 2024-12-05T00:42:31,001 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:31,001 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359411292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359411292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359411301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,378 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e255bd125780492ba842b857eca0521e 2024-12-05T00:42:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/770f9064579d4dafad37109a368efbf2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/770f9064579d4dafad37109a368efbf2 2024-12-05T00:42:31,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/770f9064579d4dafad37109a368efbf2, entries=150, sequenceid=456, filesize=12.0 K 2024-12-05T00:42:31,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/a64407b308234d7894d347291fdf1097 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/a64407b308234d7894d347291fdf1097 2024-12-05T00:42:31,394 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/a64407b308234d7894d347291fdf1097, entries=150, sequenceid=456, filesize=12.0 K 2024-12-05T00:42:31,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e255bd125780492ba842b857eca0521e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e255bd125780492ba842b857eca0521e 2024-12-05T00:42:31,403 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e255bd125780492ba842b857eca0521e, entries=150, sequenceid=456, filesize=12.0 K 2024-12-05T00:42:31,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8052c0b234f27c5e419e64636802bdde in 1001ms, sequenceid=456, compaction requested=false 2024-12-05T00:42:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-05T00:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-05T00:42:31,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-05T00:42:31,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-12-05T00:42:31,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7790 sec 2024-12-05T00:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:31,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:42:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:31,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:31,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/c7712abaeabc4be69ffe231e1f57c7b4 is 50, key is test_row_0/A:col10/1733359351674/Put/seqid=0 2024-12-05T00:42:31,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742145_1321 (size=12301) 2024-12-05T00:42:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359351747, max timestamp=1733359351754Num uniques 859; 2024-12-05T00:42:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359351747, max timestamp=1733359351754Num uniques 859; 2024-12-05T00:42:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:31,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359351747, max timestamp=1733359351754Num uniques 859; 2024-12-05T00:42:31,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:42:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T00:42:31,782 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:31,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-05T00:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T00:42:31,786 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:31,788 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:31,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:31,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359411807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359411816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359411818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359411819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359411820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T00:42:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359411922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359411933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359411933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359411933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:31,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-05T00:42:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:31,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-05T00:42:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T00:42:32,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/c7712abaeabc4be69ffe231e1f57c7b4 2024-12-05T00:42:32,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/532a263d59d44568b763c1b9a2e4d856 is 50, key is test_row_0/B:col10/1733359351674/Put/seqid=0 2024-12-05T00:42:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359412132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359412142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359412142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359412142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742146_1322 (size=12301) 2024-12-05T00:42:32,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/532a263d59d44568b763c1b9a2e4d856 2024-12-05T00:42:32,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/d7b8efb78729480498abe7724248fb13 is 50, key is test_row_0/C:col10/1733359351674/Put/seqid=0 2024-12-05T00:42:32,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742147_1323 (size=12301) 2024-12-05T00:42:32,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-05T00:42:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:32,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-05T00:42:32,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T00:42:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359412442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-05T00:42:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:32,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/d7b8efb78729480498abe7724248fb13 2024-12-05T00:42:32,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/c7712abaeabc4be69ffe231e1f57c7b4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c7712abaeabc4be69ffe231e1f57c7b4 2024-12-05T00:42:32,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c7712abaeabc4be69ffe231e1f57c7b4, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T00:42:32,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/532a263d59d44568b763c1b9a2e4d856 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/532a263d59d44568b763c1b9a2e4d856 2024-12-05T00:42:32,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/532a263d59d44568b763c1b9a2e4d856, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T00:42:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/d7b8efb78729480498abe7724248fb13 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/d7b8efb78729480498abe7724248fb13 2024-12-05T00:42:32,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/d7b8efb78729480498abe7724248fb13, entries=150, sequenceid=469, filesize=12.0 K 2024-12-05T00:42:32,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8052c0b234f27c5e419e64636802bdde in 926ms, sequenceid=469, compaction requested=true 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:32,604 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:32,604 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:32,605 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,605 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,605 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/592b50c55aa34635b4343d272b7afcbc, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/a64407b308234d7894d347291fdf1097, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/532a263d59d44568b763c1b9a2e4d856] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=36.9 K 2024-12-05T00:42:32,605 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/575033a3bbf14dde962fcde599ad4fe1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/770f9064579d4dafad37109a368efbf2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c7712abaeabc4be69ffe231e1f57c7b4] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=36.9 K 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575033a3bbf14dde962fcde599ad4fe1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733359347206 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 592b50c55aa34635b4343d272b7afcbc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733359347206 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a64407b308234d7894d347291fdf1097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733359349460 2024-12-05T00:42:32,605 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770f9064579d4dafad37109a368efbf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733359349460 2024-12-05T00:42:32,606 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 532a263d59d44568b763c1b9a2e4d856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733359350659 2024-12-05T00:42:32,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7712abaeabc4be69ffe231e1f57c7b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733359350659 2024-12-05T00:42:32,616 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:32,616 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:32,617 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/45de2cb06b514b7d92186a4226a1d228 is 50, key is test_row_0/A:col10/1733359351674/Put/seqid=0 2024-12-05T00:42:32,617 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/b7f6f48a36af45178f3742b6793ba618 is 50, key is test_row_0/B:col10/1733359351674/Put/seqid=0 2024-12-05T00:42:32,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742149_1325 (size=13323) 2024-12-05T00:42:32,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742148_1324 (size=13323) 2024-12-05T00:42:32,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:32,712 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:32,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:32,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/347e4316841a42d8a0902185d504bbf4 is 50, key is test_row_0/A:col10/1733359351811/Put/seqid=0 2024-12-05T00:42:32,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742150_1326 (size=12301) 2024-12-05T00:42:32,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:32,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359412872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T00:42:32,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359412951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359412961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359412961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359412961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:32,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359412982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:33,027 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/b7f6f48a36af45178f3742b6793ba618 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/b7f6f48a36af45178f3742b6793ba618 2024-12-05T00:42:33,035 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/45de2cb06b514b7d92186a4226a1d228 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/45de2cb06b514b7d92186a4226a1d228 2024-12-05T00:42:33,035 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into b7f6f48a36af45178f3742b6793ba618(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:33,035 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:33,035 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359352604; duration=0sec 2024-12-05T00:42:33,036 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:33,036 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:33,036 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:33,040 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:33,040 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:33,040 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:33,040 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1660f882050f4d188ce3e60ea0cb56e7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e255bd125780492ba842b857eca0521e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/d7b8efb78729480498abe7724248fb13] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=36.9 K 2024-12-05T00:42:33,042 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 1660f882050f4d188ce3e60ea0cb56e7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733359347206 2024-12-05T00:42:33,043 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e255bd125780492ba842b857eca0521e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733359349460 2024-12-05T00:42:33,043 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b8efb78729480498abe7724248fb13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733359350659 2024-12-05T00:42:33,046 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into 45de2cb06b514b7d92186a4226a1d228(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:33,046 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:33,046 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359352604; duration=0sec 2024-12-05T00:42:33,046 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:33,046 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:33,058 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:33,058 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/eef18975ada64295b6e290fc7cdf58d5 is 50, key is test_row_0/C:col10/1733359351674/Put/seqid=0 2024-12-05T00:42:33,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742151_1327 (size=13323) 2024-12-05T00:42:33,120 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/347e4316841a42d8a0902185d504bbf4 2024-12-05T00:42:33,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/bcfe459363544464be494c5da419e59e is 50, key is test_row_0/B:col10/1733359351811/Put/seqid=0 2024-12-05T00:42:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742152_1328 (size=12301) 2024-12-05T00:42:33,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359413192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:33,469 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/eef18975ada64295b6e290fc7cdf58d5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/eef18975ada64295b6e290fc7cdf58d5 2024-12-05T00:42:33,473 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into eef18975ada64295b6e290fc7cdf58d5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:33,473 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:33,473 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359352604; duration=0sec 2024-12-05T00:42:33,473 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:33,473 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359413501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:33,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/bcfe459363544464be494c5da419e59e 2024-12-05T00:42:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/cab0da7003e04dafbdd5620cc3308d32 is 50, key is test_row_0/C:col10/1733359351811/Put/seqid=0 2024-12-05T00:42:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742153_1329 (size=12301) 2024-12-05T00:42:33,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/cab0da7003e04dafbdd5620cc3308d32 2024-12-05T00:42:33,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/347e4316841a42d8a0902185d504bbf4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/347e4316841a42d8a0902185d504bbf4 2024-12-05T00:42:33,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/347e4316841a42d8a0902185d504bbf4, entries=150, sequenceid=506, filesize=12.0 K 2024-12-05T00:42:33,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/bcfe459363544464be494c5da419e59e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bcfe459363544464be494c5da419e59e 2024-12-05T00:42:33,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bcfe459363544464be494c5da419e59e, entries=150, sequenceid=506, filesize=12.0 K 2024-12-05T00:42:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/cab0da7003e04dafbdd5620cc3308d32 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cab0da7003e04dafbdd5620cc3308d32 2024-12-05T00:42:33,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cab0da7003e04dafbdd5620cc3308d32, entries=150, sequenceid=506, filesize=12.0 K 2024-12-05T00:42:33,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8052c0b234f27c5e419e64636802bdde in 857ms, sequenceid=506, compaction requested=false 2024-12-05T00:42:33,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:33,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:33,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-05T00:42:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-05T00:42:33,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-05T00:42:33,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7830 sec 2024-12-05T00:42:33,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7920 sec 2024-12-05T00:42:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-05T00:42:33,932 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:33,932 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-05T00:42:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:33,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:33,935 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:33,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:33,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:42:33,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:33,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:33,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:33,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:33,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:33,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:33,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/bbf2571e2d2a46f5b9c5ff180b8a6d37 is 50, key is test_row_0/A:col10/1733359352860/Put/seqid=0 2024-12-05T00:42:33,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742154_1330 (size=14741) 2024-12-05T00:42:34,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359413998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359353994, max timestamp=1733359354002Num uniques 859; 2024-12-05T00:42:34,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:34,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359353994, max timestamp=1733359354002Num uniques 859; 2024-12-05T00:42:34,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:34,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359353994, max timestamp=1733359354002Num uniques 859; 2024-12-05T00:42:34,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359414017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359414019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359414019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359414021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:34,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359414112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359414131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359414131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359414131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359414132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:34,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:34,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359414321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359414341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359414342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359414342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359414342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/bbf2571e2d2a46f5b9c5ff180b8a6d37 2024-12-05T00:42:34,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f538e16ab5f54f56875450c3d7dae691 is 50, key is test_row_0/B:col10/1733359352860/Put/seqid=0 2024-12-05T00:42:34,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742155_1331 (size=12301) 2024-12-05T00:42:34,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:34,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:34,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:34,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,604 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-12-05T00:42:34,605 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:34,605 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,605 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:34,607 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-12-05T00:42:34,607 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:34,607 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,607 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:34,609 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-12-05T00:42:34,609 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:34,609 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,609 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:34,609 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-12-05T00:42:34,610 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:34,610 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,610 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:34,611 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-12-05T00:42:34,611 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:34,611 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,611 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:34,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:34,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359414631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359414652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359414652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359414652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359414652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:34,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f538e16ab5f54f56875450c3d7dae691 2024-12-05T00:42:34,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/71c08e38824547f88d67cc52602b1491 is 50, key is test_row_0/C:col10/1733359352860/Put/seqid=0 2024-12-05T00:42:34,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742156_1332 (size=12301) 2024-12-05T00:42:34,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:34,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:35,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42478 deadline: 1733359415141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42526 deadline: 1733359415162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42462 deadline: 1733359415162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42474 deadline: 1733359415162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42520 deadline: 1733359415162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:42:35,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:35,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:35,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/71c08e38824547f88d67cc52602b1491 2024-12-05T00:42:35,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/bbf2571e2d2a46f5b9c5ff180b8a6d37 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bbf2571e2d2a46f5b9c5ff180b8a6d37 2024-12-05T00:42:35,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bbf2571e2d2a46f5b9c5ff180b8a6d37, entries=200, sequenceid=520, filesize=14.4 K 2024-12-05T00:42:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/f538e16ab5f54f56875450c3d7dae691 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f538e16ab5f54f56875450c3d7dae691 2024-12-05T00:42:35,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f538e16ab5f54f56875450c3d7dae691, entries=150, sequenceid=520, filesize=12.0 K 2024-12-05T00:42:35,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/71c08e38824547f88d67cc52602b1491 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/71c08e38824547f88d67cc52602b1491 2024-12-05T00:42:35,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/71c08e38824547f88d67cc52602b1491, entries=150, sequenceid=520, filesize=12.0 K 2024-12-05T00:42:35,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8052c0b234f27c5e419e64636802bdde in 1268ms, sequenceid=520, compaction requested=true 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:35,241 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8052c0b234f27c5e419e64636802bdde:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:35,241 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:35,242 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:35,242 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:35,242 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/A is initiating minor compaction (all files) 2024-12-05T00:42:35,242 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/B is initiating minor compaction (all files) 2024-12-05T00:42:35,242 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/A in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,242 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/B in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,243 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/b7f6f48a36af45178f3742b6793ba618, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bcfe459363544464be494c5da419e59e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f538e16ab5f54f56875450c3d7dae691] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=37.0 K 2024-12-05T00:42:35,243 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/45de2cb06b514b7d92186a4226a1d228, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/347e4316841a42d8a0902185d504bbf4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bbf2571e2d2a46f5b9c5ff180b8a6d37] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=39.4 K 2024-12-05T00:42:35,243 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45de2cb06b514b7d92186a4226a1d228, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733359350659 2024-12-05T00:42:35,243 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b7f6f48a36af45178f3742b6793ba618, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733359350659 2024-12-05T00:42:35,243 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfe459363544464be494c5da419e59e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733359351780 2024-12-05T00:42:35,243 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 347e4316841a42d8a0902185d504bbf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733359351780 2024-12-05T00:42:35,244 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting f538e16ab5f54f56875450c3d7dae691, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733359352846 2024-12-05T00:42:35,244 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf2571e2d2a46f5b9c5ff180b8a6d37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733359352846 2024-12-05T00:42:35,250 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:35,251 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/fbd8f33bae65487787f90ec66ed213be is 50, key is test_row_0/B:col10/1733359352860/Put/seqid=0 2024-12-05T00:42:35,252 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#A#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:35,252 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/cf4150f3366443efbbe5296e7414a495 is 50, key is test_row_0/A:col10/1733359352860/Put/seqid=0 2024-12-05T00:42:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742157_1333 (size=13425) 2024-12-05T00:42:35,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742158_1334 (size=13425) 2024-12-05T00:42:35,259 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/cf4150f3366443efbbe5296e7414a495 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/cf4150f3366443efbbe5296e7414a495 2024-12-05T00:42:35,262 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/A of 8052c0b234f27c5e419e64636802bdde into cf4150f3366443efbbe5296e7414a495(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:35,262 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:35,263 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/A, priority=13, startTime=1733359355241; duration=0sec 2024-12-05T00:42:35,263 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:35,263 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:A 2024-12-05T00:42:35,263 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:35,263 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:35,264 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 8052c0b234f27c5e419e64636802bdde/C is initiating minor compaction (all files) 2024-12-05T00:42:35,264 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8052c0b234f27c5e419e64636802bdde/C in TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,264 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/eef18975ada64295b6e290fc7cdf58d5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cab0da7003e04dafbdd5620cc3308d32, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/71c08e38824547f88d67cc52602b1491] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp, totalSize=37.0 K 2024-12-05T00:42:35,264 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting eef18975ada64295b6e290fc7cdf58d5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733359350659 2024-12-05T00:42:35,264 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab0da7003e04dafbdd5620cc3308d32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733359351780 2024-12-05T00:42:35,264 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c08e38824547f88d67cc52602b1491, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733359352846 2024-12-05T00:42:35,269 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8052c0b234f27c5e419e64636802bdde#C#compaction#290 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:35,270 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e240dde05215406abcaa5ff5e31cb305 is 50, key is test_row_0/C:col10/1733359352860/Put/seqid=0 2024-12-05T00:42:35,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742159_1335 (size=13425) 2024-12-05T00:42:35,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-05T00:42:35,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:35,337 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-05T00:42:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:35,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/04c1f5c48dde48a2a26741c62ff16909 is 50, key is test_row_0/A:col10/1733359354016/Put/seqid=0 2024-12-05T00:42:35,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742160_1336 (size=12301) 2024-12-05T00:42:35,591 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T00:42:35,667 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/fbd8f33bae65487787f90ec66ed213be as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/fbd8f33bae65487787f90ec66ed213be 2024-12-05T00:42:35,672 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/B of 8052c0b234f27c5e419e64636802bdde into fbd8f33bae65487787f90ec66ed213be(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:35,672 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:35,672 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/B, priority=13, startTime=1733359355241; duration=0sec 2024-12-05T00:42:35,672 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:35,672 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:B 2024-12-05T00:42:35,678 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/e240dde05215406abcaa5ff5e31cb305 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e240dde05215406abcaa5ff5e31cb305 2024-12-05T00:42:35,681 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8052c0b234f27c5e419e64636802bdde/C of 8052c0b234f27c5e419e64636802bdde into e240dde05215406abcaa5ff5e31cb305(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:35,681 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:35,681 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., storeName=8052c0b234f27c5e419e64636802bdde/C, priority=13, startTime=1733359355241; duration=0sec 2024-12-05T00:42:35,682 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:35,682 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8052c0b234f27c5e419e64636802bdde:C 2024-12-05T00:42:35,752 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/04c1f5c48dde48a2a26741c62ff16909 2024-12-05T00:42:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/6d0687b7fee24c1285dc06e4b07c7481 is 50, key is test_row_0/B:col10/1733359354016/Put/seqid=0 2024-12-05T00:42:35,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742161_1337 (size=12301) 2024-12-05T00:42:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:36,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. as already flushing 2024-12-05T00:42:36,163 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-12-05T00:42:36,163 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:36,163 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,164 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:36,170 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/6d0687b7fee24c1285dc06e4b07c7481 2024-12-05T00:42:36,173 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-12-05T00:42:36,174 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:36,174 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,174 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:36,174 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-12-05T00:42:36,174 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:36,174 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,174 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:36,174 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-12-05T00:42:36,174 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:36,174 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,174 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:36,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/1e8abc47256d4d1f8cf926d98a97d0e8 is 50, key is test_row_0/C:col10/1733359354016/Put/seqid=0 2024-12-05T00:42:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742162_1338 (size=12301) 2024-12-05T00:42:36,183 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-12-05T00:42:36,183 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:42:36,183 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,183 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:36,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:36,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/1e8abc47256d4d1f8cf926d98a97d0e8 2024-12-05T00:42:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/04c1f5c48dde48a2a26741c62ff16909 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04c1f5c48dde48a2a26741c62ff16909 2024-12-05T00:42:36,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04c1f5c48dde48a2a26741c62ff16909, entries=150, sequenceid=555, filesize=12.0 K 2024-12-05T00:42:36,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/6d0687b7fee24c1285dc06e4b07c7481 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/6d0687b7fee24c1285dc06e4b07c7481 2024-12-05T00:42:36,600 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/6d0687b7fee24c1285dc06e4b07c7481, entries=150, sequenceid=555, filesize=12.0 K 2024-12-05T00:42:36,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/1e8abc47256d4d1f8cf926d98a97d0e8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1e8abc47256d4d1f8cf926d98a97d0e8 2024-12-05T00:42:36,605 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1e8abc47256d4d1f8cf926d98a97d0e8, entries=150, sequenceid=555, filesize=12.0 K 2024-12-05T00:42:36,606 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=33.54 KB/34350 for 8052c0b234f27c5e419e64636802bdde in 1269ms, sequenceid=555, compaction requested=false 2024-12-05T00:42:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 8052c0b234f27c5e419e64636802bdde: 2024-12-05T00:42:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-05T00:42:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-05T00:42:36,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-05T00:42:36,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6730 sec 2024-12-05T00:42:36,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.6770 sec 2024-12-05T00:42:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T00:42:38,102 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3485 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3348 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3425 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3488 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3379 2024-12-05T00:42:38,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T00:42:38,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:42:38,103 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:42:38,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:38,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:38,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:42:38,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T00:42:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T00:42:38,106 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359358106"}]},"ts":"1733359358106"} 2024-12-05T00:42:38,107 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T00:42:38,107 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T00:42:38,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:42:38,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, UNASSIGN}] 2024-12-05T00:42:38,110 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, UNASSIGN 2024-12-05T00:42:38,110 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=8052c0b234f27c5e419e64636802bdde, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:38,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, UNASSIGN because future has completed 2024-12-05T00:42:38,111 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:42:38,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:42:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T00:42:38,265 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:38,265 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:42:38,265 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 8052c0b234f27c5e419e64636802bdde, disabling compactions & flushes 2024-12-05T00:42:38,265 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:38,265 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:38,265 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. after waiting 0 ms 2024-12-05T00:42:38,265 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:38,266 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 8052c0b234f27c5e419e64636802bdde 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T00:42:38,266 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=A 2024-12-05T00:42:38,266 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:38,266 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=B 2024-12-05T00:42:38,266 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:38,266 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8052c0b234f27c5e419e64636802bdde, store=C 2024-12-05T00:42:38,267 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:38,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/0cab6174d41e4822ac184c9b29fd854c is 50, key is test_row_0/A:col10/1733359356182/Put/seqid=0 2024-12-05T00:42:38,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742163_1339 (size=12301) 2024-12-05T00:42:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T00:42:38,682 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/0cab6174d41e4822ac184c9b29fd854c 2024-12-05T00:42:38,694 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9d9f102170a0425595ff00e229061baa is 50, key is test_row_0/B:col10/1733359356182/Put/seqid=0 2024-12-05T00:42:38,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742164_1340 (size=12301) 2024-12-05T00:42:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T00:42:39,099 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9d9f102170a0425595ff00e229061baa 2024-12-05T00:42:39,133 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/01e3c984e8694ffa982c16fa683010fe is 50, key is test_row_0/C:col10/1733359356182/Put/seqid=0 2024-12-05T00:42:39,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742165_1341 (size=12301) 2024-12-05T00:42:39,136 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/01e3c984e8694ffa982c16fa683010fe 2024-12-05T00:42:39,140 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/A/0cab6174d41e4822ac184c9b29fd854c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/0cab6174d41e4822ac184c9b29fd854c 2024-12-05T00:42:39,143 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/0cab6174d41e4822ac184c9b29fd854c, entries=150, sequenceid=565, filesize=12.0 K 2024-12-05T00:42:39,144 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/B/9d9f102170a0425595ff00e229061baa as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9d9f102170a0425595ff00e229061baa 2024-12-05T00:42:39,147 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9d9f102170a0425595ff00e229061baa, entries=150, sequenceid=565, filesize=12.0 K 2024-12-05T00:42:39,148 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/.tmp/C/01e3c984e8694ffa982c16fa683010fe as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/01e3c984e8694ffa982c16fa683010fe 2024-12-05T00:42:39,151 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/01e3c984e8694ffa982c16fa683010fe, entries=150, sequenceid=565, filesize=12.0 K 2024-12-05T00:42:39,152 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8052c0b234f27c5e419e64636802bdde in 887ms, sequenceid=565, compaction requested=true 2024-12-05T00:42:39,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c582d8e0fe404440995682927a3e8f97, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/9d2ead3fe4ad4044a3659f28c46ba8e4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/be911a0ba679469da1c0fd49957ecec0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/10b4d9a8098a4cf9b4526910f2daf602, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/4057f8748c154477a835db6e37af97b9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/75ccd82bfb8542dba754bbab2a07bdf0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bb4a161202ff4b73a117c77e7ec14059, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/d45f46b713704d3d9605cf82e9c59e3a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/ff0a051d0355497fab01bccfa4227562, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a6e6e53ef0054dc3b94f411163af7316, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/639430b1ec4e4715a444ed552dd73fd2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/622be16a6bb54e839cd8be3d7bbd8aa7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a5a8a301c9a7411e968f8dba6e144879, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04a71f20b04f4f25a28852ce7a412561, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/462c5d97589c4632be55ebaf744a42b6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/f3ed6a5b0ec4460f8eb2026f57a8277c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e6d7598766804f1c8cf69703f55201cf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/88b136a058b747fab387c94cad68a947, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e76e2f4421e04c1980574397504732c5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/21189ce66ef0476fb2953288881def61, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2a1cc77206ed49cc8791dd9649b51b13, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8871e3388c374e8996cff8f3eb5c22a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/abd0dfaf7439417088c12b09780bcb05, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/5ce8700cf6324c4eaa3dc93dacccc52f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2771508870fb4925bfc02fa7d8643212, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/3f2839085e4f49d39d73e20f4d20280d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/575033a3bbf14dde962fcde599ad4fe1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8bdb68c6c9664e3986ad4a21f6937cdf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/770f9064579d4dafad37109a368efbf2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/45de2cb06b514b7d92186a4226a1d228, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c7712abaeabc4be69ffe231e1f57c7b4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/347e4316841a42d8a0902185d504bbf4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bbf2571e2d2a46f5b9c5ff180b8a6d37] to archive 2024-12-05T00:42:39,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:42:39,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c582d8e0fe404440995682927a3e8f97 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c582d8e0fe404440995682927a3e8f97 2024-12-05T00:42:39,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/9d2ead3fe4ad4044a3659f28c46ba8e4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/9d2ead3fe4ad4044a3659f28c46ba8e4 2024-12-05T00:42:39,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/be911a0ba679469da1c0fd49957ecec0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/be911a0ba679469da1c0fd49957ecec0 2024-12-05T00:42:39,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/10b4d9a8098a4cf9b4526910f2daf602 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/10b4d9a8098a4cf9b4526910f2daf602 2024-12-05T00:42:39,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/4057f8748c154477a835db6e37af97b9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/4057f8748c154477a835db6e37af97b9 2024-12-05T00:42:39,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/75ccd82bfb8542dba754bbab2a07bdf0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/75ccd82bfb8542dba754bbab2a07bdf0 2024-12-05T00:42:39,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bb4a161202ff4b73a117c77e7ec14059 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bb4a161202ff4b73a117c77e7ec14059 2024-12-05T00:42:39,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/d45f46b713704d3d9605cf82e9c59e3a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/d45f46b713704d3d9605cf82e9c59e3a 2024-12-05T00:42:39,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/ff0a051d0355497fab01bccfa4227562 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/ff0a051d0355497fab01bccfa4227562 2024-12-05T00:42:39,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a6e6e53ef0054dc3b94f411163af7316 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a6e6e53ef0054dc3b94f411163af7316 2024-12-05T00:42:39,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/639430b1ec4e4715a444ed552dd73fd2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/639430b1ec4e4715a444ed552dd73fd2 2024-12-05T00:42:39,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/622be16a6bb54e839cd8be3d7bbd8aa7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/622be16a6bb54e839cd8be3d7bbd8aa7 2024-12-05T00:42:39,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a5a8a301c9a7411e968f8dba6e144879 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/a5a8a301c9a7411e968f8dba6e144879 2024-12-05T00:42:39,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04a71f20b04f4f25a28852ce7a412561 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04a71f20b04f4f25a28852ce7a412561 2024-12-05T00:42:39,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/462c5d97589c4632be55ebaf744a42b6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/462c5d97589c4632be55ebaf744a42b6 2024-12-05T00:42:39,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/f3ed6a5b0ec4460f8eb2026f57a8277c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/f3ed6a5b0ec4460f8eb2026f57a8277c 2024-12-05T00:42:39,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e6d7598766804f1c8cf69703f55201cf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e6d7598766804f1c8cf69703f55201cf 2024-12-05T00:42:39,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/88b136a058b747fab387c94cad68a947 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/88b136a058b747fab387c94cad68a947 2024-12-05T00:42:39,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e76e2f4421e04c1980574397504732c5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/e76e2f4421e04c1980574397504732c5 2024-12-05T00:42:39,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/21189ce66ef0476fb2953288881def61 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/21189ce66ef0476fb2953288881def61 2024-12-05T00:42:39,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2a1cc77206ed49cc8791dd9649b51b13 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2a1cc77206ed49cc8791dd9649b51b13 2024-12-05T00:42:39,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8871e3388c374e8996cff8f3eb5c22a7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8871e3388c374e8996cff8f3eb5c22a7 2024-12-05T00:42:39,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/abd0dfaf7439417088c12b09780bcb05 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/abd0dfaf7439417088c12b09780bcb05 2024-12-05T00:42:39,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/5ce8700cf6324c4eaa3dc93dacccc52f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/5ce8700cf6324c4eaa3dc93dacccc52f 2024-12-05T00:42:39,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2771508870fb4925bfc02fa7d8643212 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/2771508870fb4925bfc02fa7d8643212 2024-12-05T00:42:39,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/3f2839085e4f49d39d73e20f4d20280d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/3f2839085e4f49d39d73e20f4d20280d 2024-12-05T00:42:39,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/575033a3bbf14dde962fcde599ad4fe1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/575033a3bbf14dde962fcde599ad4fe1 2024-12-05T00:42:39,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8bdb68c6c9664e3986ad4a21f6937cdf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/8bdb68c6c9664e3986ad4a21f6937cdf 2024-12-05T00:42:39,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/770f9064579d4dafad37109a368efbf2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/770f9064579d4dafad37109a368efbf2 2024-12-05T00:42:39,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/45de2cb06b514b7d92186a4226a1d228 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/45de2cb06b514b7d92186a4226a1d228 2024-12-05T00:42:39,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c7712abaeabc4be69ffe231e1f57c7b4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/c7712abaeabc4be69ffe231e1f57c7b4 2024-12-05T00:42:39,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/347e4316841a42d8a0902185d504bbf4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/347e4316841a42d8a0902185d504bbf4 2024-12-05T00:42:39,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bbf2571e2d2a46f5b9c5ff180b8a6d37 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/bbf2571e2d2a46f5b9c5ff180b8a6d37 2024-12-05T00:42:39,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9c3638b8df6e4d5781d52b6cf9980f52, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c3fe635ac8e74d34b8935581858edadf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0dab45a336ab4ce3bee05bdcb50f14b6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c222781b3492430fa5eaa96da28d4a87, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0fd4c6b19c66462cbc8636f6631d2ca0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f1272edd5c5846e485f3ab9de619b612, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bd3b1b23b5114dfbbc2fa5dd36a933b7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/147db0ff33d741ee9db7adb224949fbf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/77a6b01da7f84659ac19bcd2359f1c95, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/e7e4ac0fad0443b1b04ef6c7645cd839, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/dbea2e8e8c1d404fa028a14eb83c0268, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f501d8216b2f4c58978de5c223490ab6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/8a8bf08bb0ca46b693afb52393718132, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/00e17e92edaa46458d83bf24535bfb87, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c10cd996fb174264be9249f0cacfbd86, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/774b5e6450c44dbd8e70ce1af2958aff, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/973429f57e104741b7e3d09d98d61a27, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/af2eb688f90b463bb8f0f36367ea323c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/1dbfbf39cd1e4b57959f9032642786bf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/27603056f58c40ba96504e6864023fed, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/eba7d93972154c04a75bc3e78866b89d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/94b5e09f9364441f876c70f2fb7ae8ac, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/87f0c121c3384a6db0f774a8690a0d6d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9dbbb32cc41543be9e841b21733d50b1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0abf18449c9046768c28a87e55c157e0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/cf7f4baddca64bca872afd58c702a76e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/592b50c55aa34635b4343d272b7afcbc, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0663279c3e384927a70bba11a616285e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/a64407b308234d7894d347291fdf1097, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/b7f6f48a36af45178f3742b6793ba618, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/532a263d59d44568b763c1b9a2e4d856, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bcfe459363544464be494c5da419e59e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f538e16ab5f54f56875450c3d7dae691] to archive 2024-12-05T00:42:39,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:42:39,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9c3638b8df6e4d5781d52b6cf9980f52 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9c3638b8df6e4d5781d52b6cf9980f52 2024-12-05T00:42:39,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c3fe635ac8e74d34b8935581858edadf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c3fe635ac8e74d34b8935581858edadf 2024-12-05T00:42:39,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0dab45a336ab4ce3bee05bdcb50f14b6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0dab45a336ab4ce3bee05bdcb50f14b6 2024-12-05T00:42:39,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c222781b3492430fa5eaa96da28d4a87 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c222781b3492430fa5eaa96da28d4a87 2024-12-05T00:42:39,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0fd4c6b19c66462cbc8636f6631d2ca0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0fd4c6b19c66462cbc8636f6631d2ca0 2024-12-05T00:42:39,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f1272edd5c5846e485f3ab9de619b612 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f1272edd5c5846e485f3ab9de619b612 2024-12-05T00:42:39,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bd3b1b23b5114dfbbc2fa5dd36a933b7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bd3b1b23b5114dfbbc2fa5dd36a933b7 2024-12-05T00:42:39,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/147db0ff33d741ee9db7adb224949fbf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/147db0ff33d741ee9db7adb224949fbf 2024-12-05T00:42:39,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/77a6b01da7f84659ac19bcd2359f1c95 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/77a6b01da7f84659ac19bcd2359f1c95 2024-12-05T00:42:39,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/e7e4ac0fad0443b1b04ef6c7645cd839 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/e7e4ac0fad0443b1b04ef6c7645cd839 2024-12-05T00:42:39,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/dbea2e8e8c1d404fa028a14eb83c0268 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/dbea2e8e8c1d404fa028a14eb83c0268 2024-12-05T00:42:39,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f501d8216b2f4c58978de5c223490ab6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f501d8216b2f4c58978de5c223490ab6 2024-12-05T00:42:39,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/8a8bf08bb0ca46b693afb52393718132 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/8a8bf08bb0ca46b693afb52393718132 2024-12-05T00:42:39,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/00e17e92edaa46458d83bf24535bfb87 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/00e17e92edaa46458d83bf24535bfb87 2024-12-05T00:42:39,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c10cd996fb174264be9249f0cacfbd86 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/c10cd996fb174264be9249f0cacfbd86 2024-12-05T00:42:39,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/774b5e6450c44dbd8e70ce1af2958aff to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/774b5e6450c44dbd8e70ce1af2958aff 2024-12-05T00:42:39,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/973429f57e104741b7e3d09d98d61a27 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/973429f57e104741b7e3d09d98d61a27 2024-12-05T00:42:39,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/af2eb688f90b463bb8f0f36367ea323c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/af2eb688f90b463bb8f0f36367ea323c 2024-12-05T00:42:39,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/1dbfbf39cd1e4b57959f9032642786bf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/1dbfbf39cd1e4b57959f9032642786bf 2024-12-05T00:42:39,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/27603056f58c40ba96504e6864023fed to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/27603056f58c40ba96504e6864023fed 2024-12-05T00:42:39,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/eba7d93972154c04a75bc3e78866b89d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/eba7d93972154c04a75bc3e78866b89d 2024-12-05T00:42:39,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/94b5e09f9364441f876c70f2fb7ae8ac to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/94b5e09f9364441f876c70f2fb7ae8ac 2024-12-05T00:42:39,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/87f0c121c3384a6db0f774a8690a0d6d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/87f0c121c3384a6db0f774a8690a0d6d 2024-12-05T00:42:39,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9dbbb32cc41543be9e841b21733d50b1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9dbbb32cc41543be9e841b21733d50b1 2024-12-05T00:42:39,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0abf18449c9046768c28a87e55c157e0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0abf18449c9046768c28a87e55c157e0 2024-12-05T00:42:39,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/cf7f4baddca64bca872afd58c702a76e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/cf7f4baddca64bca872afd58c702a76e 2024-12-05T00:42:39,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/592b50c55aa34635b4343d272b7afcbc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/592b50c55aa34635b4343d272b7afcbc 2024-12-05T00:42:39,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0663279c3e384927a70bba11a616285e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/0663279c3e384927a70bba11a616285e 2024-12-05T00:42:39,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/a64407b308234d7894d347291fdf1097 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/a64407b308234d7894d347291fdf1097 2024-12-05T00:42:39,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/b7f6f48a36af45178f3742b6793ba618 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/b7f6f48a36af45178f3742b6793ba618 2024-12-05T00:42:39,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/532a263d59d44568b763c1b9a2e4d856 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/532a263d59d44568b763c1b9a2e4d856 2024-12-05T00:42:39,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bcfe459363544464be494c5da419e59e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/bcfe459363544464be494c5da419e59e 2024-12-05T00:42:39,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f538e16ab5f54f56875450c3d7dae691 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/f538e16ab5f54f56875450c3d7dae691 2024-12-05T00:42:39,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3e40c85731cc4090a608150d07627439, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/65025a84422c4bfe99f047effca91b2a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/02d95c418fec4770ab91e45eaf1bee4b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/64d50dff3f234b6789c688194f7cd576, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/4cde91b2360e499688fd29edfac75494, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/dbf950f6ebd74564898827acf2ac702f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3a184bc984134c6a9fe8874f366ae69c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/15c6bec1f05d49d4bed93c31477cf0f2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/449abbb6328e488a9e562e792be51d70, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/ffd4bb5634644ce29cf26737d2934566, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/675da4d58b5e46268d407111adcde812, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/0b19f237c6cb4d879062f91beb53f9b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cc47eed445d54984b9443412e27986b3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e53fe97bc2d44ef594f8f27d55639a55, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/093230dcd46c47b29eb01f4111a1e44c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/b1b9fea145c246018229fc29216bb291, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5e7024f2042d458a85eacdbaeb520141, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/fe9e76d9756343318455af1eba4243e3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/2e9b6ca928f24ba3b5cde857dd0308aa, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7f2d75c2d6cc4407855518269756d19c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/bb2f8ec28195486ea3f72f338c11cad1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/564d01fa0ecf4358986638fcdb685a4d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5fc9f929b096426294e52490471d9b61, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/51c7921e6f5b4cc5a1a3904c888a3508, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/39d75a46f0134c7283d75beeda79c3de, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/a09b76e672e84e80bc07e0e432128bf0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1660f882050f4d188ce3e60ea0cb56e7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7b562aae7e514622b6f7334bd2e28bd1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e255bd125780492ba842b857eca0521e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/eef18975ada64295b6e290fc7cdf58d5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/d7b8efb78729480498abe7724248fb13, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cab0da7003e04dafbdd5620cc3308d32, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/71c08e38824547f88d67cc52602b1491] to archive 2024-12-05T00:42:39,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:42:39,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3e40c85731cc4090a608150d07627439 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3e40c85731cc4090a608150d07627439 2024-12-05T00:42:39,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/65025a84422c4bfe99f047effca91b2a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/65025a84422c4bfe99f047effca91b2a 2024-12-05T00:42:39,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/02d95c418fec4770ab91e45eaf1bee4b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/02d95c418fec4770ab91e45eaf1bee4b 2024-12-05T00:42:39,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/64d50dff3f234b6789c688194f7cd576 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/64d50dff3f234b6789c688194f7cd576 2024-12-05T00:42:39,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/4cde91b2360e499688fd29edfac75494 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/4cde91b2360e499688fd29edfac75494 2024-12-05T00:42:39,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/dbf950f6ebd74564898827acf2ac702f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/dbf950f6ebd74564898827acf2ac702f 2024-12-05T00:42:39,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3a184bc984134c6a9fe8874f366ae69c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/3a184bc984134c6a9fe8874f366ae69c 2024-12-05T00:42:39,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/15c6bec1f05d49d4bed93c31477cf0f2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/15c6bec1f05d49d4bed93c31477cf0f2 2024-12-05T00:42:39,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/449abbb6328e488a9e562e792be51d70 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/449abbb6328e488a9e562e792be51d70 2024-12-05T00:42:39,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/ffd4bb5634644ce29cf26737d2934566 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/ffd4bb5634644ce29cf26737d2934566 2024-12-05T00:42:39,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/675da4d58b5e46268d407111adcde812 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/675da4d58b5e46268d407111adcde812 2024-12-05T00:42:39,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/0b19f237c6cb4d879062f91beb53f9b8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/0b19f237c6cb4d879062f91beb53f9b8 2024-12-05T00:42:39,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cc47eed445d54984b9443412e27986b3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cc47eed445d54984b9443412e27986b3 2024-12-05T00:42:39,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e53fe97bc2d44ef594f8f27d55639a55 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e53fe97bc2d44ef594f8f27d55639a55 2024-12-05T00:42:39,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/093230dcd46c47b29eb01f4111a1e44c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/093230dcd46c47b29eb01f4111a1e44c 2024-12-05T00:42:39,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/b1b9fea145c246018229fc29216bb291 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/b1b9fea145c246018229fc29216bb291 2024-12-05T00:42:39,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5e7024f2042d458a85eacdbaeb520141 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5e7024f2042d458a85eacdbaeb520141 2024-12-05T00:42:39,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/fe9e76d9756343318455af1eba4243e3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/fe9e76d9756343318455af1eba4243e3 2024-12-05T00:42:39,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/2e9b6ca928f24ba3b5cde857dd0308aa to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/2e9b6ca928f24ba3b5cde857dd0308aa 2024-12-05T00:42:39,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7f2d75c2d6cc4407855518269756d19c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7f2d75c2d6cc4407855518269756d19c 2024-12-05T00:42:39,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/bb2f8ec28195486ea3f72f338c11cad1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/bb2f8ec28195486ea3f72f338c11cad1 2024-12-05T00:42:39,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/564d01fa0ecf4358986638fcdb685a4d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/564d01fa0ecf4358986638fcdb685a4d 2024-12-05T00:42:39,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5fc9f929b096426294e52490471d9b61 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/5fc9f929b096426294e52490471d9b61 2024-12-05T00:42:39,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/51c7921e6f5b4cc5a1a3904c888a3508 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/51c7921e6f5b4cc5a1a3904c888a3508 2024-12-05T00:42:39,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/39d75a46f0134c7283d75beeda79c3de to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/39d75a46f0134c7283d75beeda79c3de 2024-12-05T00:42:39,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/a09b76e672e84e80bc07e0e432128bf0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/a09b76e672e84e80bc07e0e432128bf0 2024-12-05T00:42:39,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1660f882050f4d188ce3e60ea0cb56e7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1660f882050f4d188ce3e60ea0cb56e7 2024-12-05T00:42:39,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7b562aae7e514622b6f7334bd2e28bd1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/7b562aae7e514622b6f7334bd2e28bd1 2024-12-05T00:42:39,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e255bd125780492ba842b857eca0521e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e255bd125780492ba842b857eca0521e 2024-12-05T00:42:39,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/eef18975ada64295b6e290fc7cdf58d5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/eef18975ada64295b6e290fc7cdf58d5 2024-12-05T00:42:39,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/d7b8efb78729480498abe7724248fb13 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/d7b8efb78729480498abe7724248fb13 2024-12-05T00:42:39,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cab0da7003e04dafbdd5620cc3308d32 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/cab0da7003e04dafbdd5620cc3308d32 2024-12-05T00:42:39,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/71c08e38824547f88d67cc52602b1491 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/71c08e38824547f88d67cc52602b1491 2024-12-05T00:42:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T00:42:39,242 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/recovered.edits/568.seqid, newMaxSeqId=568, maxSeqId=1 2024-12-05T00:42:39,242 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. 2024-12-05T00:42:39,243 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 8052c0b234f27c5e419e64636802bdde: Waiting for close lock at 1733359358265Running coprocessor pre-close hooks at 1733359358265Disabling compacts and flushes for region at 1733359358265Disabling writes for close at 1733359358265Obtaining lock to block concurrent updates at 1733359358266 (+1 ms)Preparing flush snapshotting stores in 8052c0b234f27c5e419e64636802bdde at 1733359358266Finished memstore snapshotting TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733359358267 (+1 ms)Flushing stores of TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde. at 1733359358268 (+1 ms)Flushing 8052c0b234f27c5e419e64636802bdde/A: creating writer at 1733359358268Flushing 8052c0b234f27c5e419e64636802bdde/A: appending metadata at 1733359358273 (+5 ms)Flushing 8052c0b234f27c5e419e64636802bdde/A: closing flushed file at 1733359358273Flushing 8052c0b234f27c5e419e64636802bdde/B: creating writer at 1733359358690 (+417 ms)Flushing 8052c0b234f27c5e419e64636802bdde/B: appending metadata at 1733359358694 (+4 ms)Flushing 8052c0b234f27c5e419e64636802bdde/B: closing flushed file at 1733359358694Flushing 8052c0b234f27c5e419e64636802bdde/C: creating writer at 1733359359130 (+436 ms)Flushing 8052c0b234f27c5e419e64636802bdde/C: appending metadata at 1733359359132 (+2 ms)Flushing 8052c0b234f27c5e419e64636802bdde/C: closing flushed file at 1733359359133 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20d9d51c: reopening flushed file at 1733359359140 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4abc0920: reopening flushed file at 1733359359143 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f99fa9b: reopening flushed file at 1733359359147 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8052c0b234f27c5e419e64636802bdde in 887ms, sequenceid=565, compaction requested=true at 1733359359152 (+5 ms)Writing region close event to WAL at 1733359359240 (+88 ms)Running coprocessor post-close hooks at 1733359359242 (+2 ms)Closed at 1733359359242 2024-12-05T00:42:39,244 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:39,244 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=8052c0b234f27c5e419e64636802bdde, regionState=CLOSED 2024-12-05T00:42:39,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:42:39,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-05T00:42:39,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 8052c0b234f27c5e419e64636802bdde, server=fea72ea5c4b6,42683,1733359267474 in 1.1350 sec 2024-12-05T00:42:39,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-05T00:42:39,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8052c0b234f27c5e419e64636802bdde, UNASSIGN in 1.1380 sec 2024-12-05T00:42:39,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-05T00:42:39,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1410 sec 2024-12-05T00:42:39,252 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359359252"}]},"ts":"1733359359252"} 2024-12-05T00:42:39,253 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T00:42:39,253 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T00:42:39,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-05T00:42:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-05T00:42:40,253 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:40,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T00:42:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,259 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-05T00:42:40,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,263 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:40,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/recovered.edits] 2024-12-05T00:42:40,271 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04c1f5c48dde48a2a26741c62ff16909 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/04c1f5c48dde48a2a26741c62ff16909 2024-12-05T00:42:40,274 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/0cab6174d41e4822ac184c9b29fd854c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/0cab6174d41e4822ac184c9b29fd854c 2024-12-05T00:42:40,276 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/cf4150f3366443efbbe5296e7414a495 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/A/cf4150f3366443efbbe5296e7414a495 2024-12-05T00:42:40,280 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/6d0687b7fee24c1285dc06e4b07c7481 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/6d0687b7fee24c1285dc06e4b07c7481 2024-12-05T00:42:40,283 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9d9f102170a0425595ff00e229061baa to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/9d9f102170a0425595ff00e229061baa 2024-12-05T00:42:40,285 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/fbd8f33bae65487787f90ec66ed213be to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/B/fbd8f33bae65487787f90ec66ed213be 2024-12-05T00:42:40,288 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/01e3c984e8694ffa982c16fa683010fe to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/01e3c984e8694ffa982c16fa683010fe 2024-12-05T00:42:40,290 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1e8abc47256d4d1f8cf926d98a97d0e8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/1e8abc47256d4d1f8cf926d98a97d0e8 2024-12-05T00:42:40,292 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e240dde05215406abcaa5ff5e31cb305 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/C/e240dde05215406abcaa5ff5e31cb305 2024-12-05T00:42:40,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/recovered.edits/568.seqid to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde/recovered.edits/568.seqid 2024-12-05T00:42:40,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/8052c0b234f27c5e419e64636802bdde 2024-12-05T00:42:40,297 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T00:42:40,298 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,301 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T00:42:40,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T00:42:40,304 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,304 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T00:42:40,304 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733359360304"}]},"ts":"9223372036854775807"} 2024-12-05T00:42:40,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T00:42:40,305 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8052c0b234f27c5e419e64636802bdde, NAME => 'TestAcidGuarantees,,1733359332356.8052c0b234f27c5e419e64636802bdde.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T00:42:40,305 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T00:42:40,306 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733359360305"}]},"ts":"9223372036854775807"} 2024-12-05T00:42:40,307 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T00:42:40,307 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-12-05T00:42:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-05T00:42:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T00:42:40,373 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:40,382 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-866614165_22 at /127.0.0.1:37422 [Waiting for operation #392] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1522171622_22 at /127.0.0.1:35892 [Waiting for operation #273] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 329) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9401 (was 9218) - AvailableMemoryMB LEAK? - 2024-12-05T00:42:40,392 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=9401 2024-12-05T00:42:40,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:42:40,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:42:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:40,396 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:42:40,396 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:40,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-05T00:42:40,397 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:42:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T00:42:40,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742166_1342 (size=1061) 2024-12-05T00:42:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T00:42:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T00:42:40,810 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:42:40,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742167_1343 (size=53) 2024-12-05T00:42:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T00:42:41,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:42:41,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing edbe57733a3fd4ee0e06e7066e75f311, disabling compactions & flushes 2024-12-05T00:42:41,220 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. after waiting 0 ms 2024-12-05T00:42:41,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,220 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for edbe57733a3fd4ee0e06e7066e75f311: Waiting for close lock at 1733359361220Disabling compacts and flushes for region at 1733359361220Disabling writes for close at 1733359361220Writing region close event to WAL at 1733359361220Closed at 1733359361220 2024-12-05T00:42:41,222 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:42:41,222 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733359361222"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359361222"}]},"ts":"1733359361222"} 2024-12-05T00:42:41,224 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:42:41,225 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:42:41,226 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359361225"}]},"ts":"1733359361225"} 2024-12-05T00:42:41,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T00:42:41,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, ASSIGN}] 2024-12-05T00:42:41,229 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, ASSIGN 2024-12-05T00:42:41,230 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=false 2024-12-05T00:42:41,380 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:41,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, ASSIGN because future has completed 2024-12-05T00:42:41,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:42:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T00:42:41,541 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,541 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:42:41,542 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,542 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:42:41,542 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,542 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,544 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,546 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:41,546 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbe57733a3fd4ee0e06e7066e75f311 columnFamilyName A 2024-12-05T00:42:41,546 DEBUG [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:41,547 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(327): Store=edbe57733a3fd4ee0e06e7066e75f311/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:41,547 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,548 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:41,549 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbe57733a3fd4ee0e06e7066e75f311 columnFamilyName B 2024-12-05T00:42:41,549 DEBUG [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:41,550 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(327): Store=edbe57733a3fd4ee0e06e7066e75f311/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:41,550 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,551 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:41,552 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbe57733a3fd4ee0e06e7066e75f311 columnFamilyName C 2024-12-05T00:42:41,552 DEBUG [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:41,552 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(327): Store=edbe57733a3fd4ee0e06e7066e75f311/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:41,553 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,553 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,554 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,554 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,556 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,556 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,557 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:42:41,559 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,561 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:42:41,562 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened edbe57733a3fd4ee0e06e7066e75f311; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68650341, jitterRate=0.022969797253608704}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:42:41,562 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:41,562 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for edbe57733a3fd4ee0e06e7066e75f311: Running coprocessor pre-open hook at 1733359361542Writing region info on filesystem at 1733359361542Initializing all the Stores at 1733359361543 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359361544 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359361544Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359361544Cleaning up temporary data from old regions at 1733359361556 (+12 ms)Running coprocessor post-open hooks at 1733359361562 (+6 ms)Region opened successfully at 1733359361562 2024-12-05T00:42:41,563 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., pid=93, masterSystemTime=1733359361535 2024-12-05T00:42:41,565 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,565 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:41,565 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:41,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:42:41,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-05T00:42:41,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 in 185 msec 2024-12-05T00:42:41,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T00:42:41,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, ASSIGN in 341 msec 2024-12-05T00:42:41,571 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:42:41,571 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359361571"}]},"ts":"1733359361571"} 2024-12-05T00:42:41,573 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T00:42:41,573 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:42:41,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-05T00:42:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T00:42:42,554 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:42,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d468cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:42,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:42,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:42,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64274c9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:42,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11430e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:42,563 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:42,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:42,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:42:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:42:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3590e8ad 2024-12-05T00:42:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:42:42,567 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:42:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:42:42,569 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:42:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T00:42:42,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742168_1344 (size=1097) 2024-12-05T00:42:42,984 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T00:42:42,984 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T00:42:42,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:42:42,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, REOPEN/MOVE}] 2024-12-05T00:42:42,995 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, REOPEN/MOVE 2024-12-05T00:42:42,996 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:42,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, REOPEN/MOVE because future has completed 2024-12-05T00:42:42,998 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:42:42,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:42:43,151 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,151 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:42:43,151 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing edbe57733a3fd4ee0e06e7066e75f311, disabling compactions & flushes 2024-12-05T00:42:43,151 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,151 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,151 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. after waiting 0 ms 2024-12-05T00:42:43,151 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,155 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T00:42:43,156 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,156 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for edbe57733a3fd4ee0e06e7066e75f311: Waiting for close lock at 1733359363151Running coprocessor pre-close hooks at 1733359363151Disabling compacts and flushes for region at 1733359363151Disabling writes for close at 1733359363151Writing region close event to WAL at 1733359363152 (+1 ms)Running coprocessor post-close hooks at 1733359363156 (+4 ms)Closed at 1733359363156 2024-12-05T00:42:43,156 WARN [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: edbe57733a3fd4ee0e06e7066e75f311 to self. 2024-12-05T00:42:43,159 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=CLOSED 2024-12-05T00:42:43,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:42:43,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-05T00:42:43,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 in 164 msec 2024-12-05T00:42:43,166 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, REOPEN/MOVE; state=CLOSED, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=true 2024-12-05T00:42:43,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, REOPEN/MOVE because future has completed 2024-12-05T00:42:43,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:42:43,474 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,474 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:42:43,475 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,475 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:42:43,475 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,475 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,476 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,477 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:43,477 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbe57733a3fd4ee0e06e7066e75f311 columnFamilyName A 2024-12-05T00:42:43,478 DEBUG [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:43,479 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(327): Store=edbe57733a3fd4ee0e06e7066e75f311/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:43,479 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,480 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:43,480 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbe57733a3fd4ee0e06e7066e75f311 columnFamilyName B 2024-12-05T00:42:43,480 DEBUG [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:43,481 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(327): Store=edbe57733a3fd4ee0e06e7066e75f311/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:43,481 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,481 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:42:43,481 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbe57733a3fd4ee0e06e7066e75f311 columnFamilyName C 2024-12-05T00:42:43,482 DEBUG [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:43,482 INFO [StoreOpener-edbe57733a3fd4ee0e06e7066e75f311-1 {}] regionserver.HStore(327): Store=edbe57733a3fd4ee0e06e7066e75f311/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:42:43,482 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,482 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,483 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,484 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,485 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,485 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,486 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:42:43,488 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,489 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened edbe57733a3fd4ee0e06e7066e75f311; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72938874, jitterRate=0.08687391877174377}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:42:43,489 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,490 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for edbe57733a3fd4ee0e06e7066e75f311: Running coprocessor pre-open hook at 1733359363475Writing region info on filesystem at 1733359363475Initializing all the Stores at 1733359363476 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359363476Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359363476Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359363476Cleaning up temporary data from old regions at 1733359363486 (+10 ms)Running coprocessor post-open hooks at 1733359363489 (+3 ms)Region opened successfully at 1733359363490 (+1 ms) 2024-12-05T00:42:43,491 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., pid=98, masterSystemTime=1733359363471 2024-12-05T00:42:43,494 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,494 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=OPEN, openSeqNum=5, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:42:43,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-05T00:42:43,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 in 178 msec 2024-12-05T00:42:43,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T00:42:43,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, REOPEN/MOVE in 504 msec 2024-12-05T00:42:43,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-05T00:42:43,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-12-05T00:42:43,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-12-05T00:42:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-05T00:42:43,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:43,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aba547e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,507 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9474d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,509 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe7779e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@440285d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,511 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374d15d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,513 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422e0d77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c1e738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e253d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,517 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1b699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508ab67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,519 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b124711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,520 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1650eefc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d261f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,523 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f14295e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,524 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f52cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2853677b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,526 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e44a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,528 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fcd84c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d51c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,530 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3647f4d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,531 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4f80a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff4aaae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,534 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59acac9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393f4afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f15f223, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,538 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214b5f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607cd0e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b220e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:42:43,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:42:43,542 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:42:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:42:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:42:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426a866f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:42:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:42:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:42:43,544 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:42:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f0ebb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:42:43,545 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,545 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:43,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:42:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:43,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:43,572 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,572 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733359363564, max timestamp=1733359363570Num uniques 859; 2024-12-05T00:42:43,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:43,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733359363564, max timestamp=1733359363570Num uniques 859; 2024-12-05T00:42:43,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733359363564, max timestamp=1733359363570Num uniques 859; 2024-12-05T00:42:43,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,584 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,584 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b25e55cb617f4617bfa43af6c50b8dee_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359363555/Put/seqid=0 2024-12-05T00:42:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,587 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,588 DEBUG [Thread-1578 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,588 DEBUG [Thread-1578 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359423590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359423591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-05T00:42:43,594 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:43,595 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:43,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:43,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359423592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359423597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,604 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,605 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,606 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:42:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742169_1345 (size=9714) 2024-12-05T00:42:43,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359423611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,620 DEBUG [Thread-1576 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:42:43,620 DEBUG [Thread-1576 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:42:43,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:42:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:42:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359423702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359423702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359423711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359423711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359423722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:43,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:43,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:43,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:43,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:43,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359423913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359423913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359423932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359423932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359423932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:44,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b25e55cb617f4617bfa43af6c50b8dee_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b25e55cb617f4617bfa43af6c50b8dee_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:44,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/6152a928435c4f6498cd592481c4c1d7, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:44,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/6152a928435c4f6498cd592481c4c1d7 is 175, key is test_row_0/A:col10/1733359363555/Put/seqid=0 2024-12-05T00:42:44,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742170_1346 (size=22361) 2024-12-05T00:42:44,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359424222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359424232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359424241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359424241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359424241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/6152a928435c4f6498cd592481c4c1d7 2024-12-05T00:42:44,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/3c8865a5c9ef47ffa0e9dae300ea4155 is 50, key is test_row_0/B:col10/1733359363555/Put/seqid=0 2024-12-05T00:42:44,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742171_1347 (size=9657) 2024-12-05T00:42:44,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:44,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359424741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359424741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,766 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:42:44,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359424761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359424762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359424762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:44,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/3c8865a5c9ef47ffa0e9dae300ea4155 2024-12-05T00:42:44,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/964c729296c44e16af9c12d0e6115aa1 is 50, key is test_row_0/C:col10/1733359363555/Put/seqid=0 2024-12-05T00:42:44,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742172_1348 (size=9657) 2024-12-05T00:42:44,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:44,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:44,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:44,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:45,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/964c729296c44e16af9c12d0e6115aa1 2024-12-05T00:42:45,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/6152a928435c4f6498cd592481c4c1d7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7 2024-12-05T00:42:45,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7, entries=100, sequenceid=15, filesize=21.8 K 2024-12-05T00:42:45,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:45,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/3c8865a5c9ef47ffa0e9dae300ea4155 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3c8865a5c9ef47ffa0e9dae300ea4155 2024-12-05T00:42:45,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:45,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3c8865a5c9ef47ffa0e9dae300ea4155, entries=100, sequenceid=15, filesize=9.4 K 2024-12-05T00:42:45,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/964c729296c44e16af9c12d0e6115aa1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/964c729296c44e16af9c12d0e6115aa1 2024-12-05T00:42:45,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/964c729296c44e16af9c12d0e6115aa1, entries=100, sequenceid=15, filesize=9.4 K 2024-12-05T00:42:45,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for edbe57733a3fd4ee0e06e7066e75f311 in 1734ms, sequenceid=15, compaction requested=false 2024-12-05T00:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T00:42:45,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:45,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:45,432 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054c82a4097da048f28776eabb124046ee_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359363586/Put/seqid=0 2024-12-05T00:42:45,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742173_1349 (size=12154) 2024-12-05T00:42:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:45,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359425765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359425766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:45,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359425781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359425782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,789 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359425782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:45,789 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:45,790 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:45,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:45,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054c82a4097da048f28776eabb124046ee_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054c82a4097da048f28776eabb124046ee_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/4268534fa050461bb6313fb31e58652f, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/4268534fa050461bb6313fb31e58652f is 175, key is test_row_0/A:col10/1733359363586/Put/seqid=0 2024-12-05T00:42:45,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742174_1350 (size=30955) 2024-12-05T00:42:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359425871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:45,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359425882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359426091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359426091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:46,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/4268534fa050461bb6313fb31e58652f 2024-12-05T00:42:46,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/fb0b5d2f733c4442a97f9cba481be9a7 is 50, key is test_row_0/B:col10/1733359363586/Put/seqid=0 2024-12-05T00:42:46,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742175_1351 (size=12001) 2024-12-05T00:42:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359426401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359426402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:46,660 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/fb0b5d2f733c4442a97f9cba481be9a7 2024-12-05T00:42:46,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/565772b12cd344f682a8039a73d14f7b is 50, key is test_row_0/C:col10/1733359363586/Put/seqid=0 2024-12-05T00:42:46,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742176_1352 (size=12001) 2024-12-05T00:42:46,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359426911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359426911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:47,070 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/565772b12cd344f682a8039a73d14f7b 2024-12-05T00:42:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/4268534fa050461bb6313fb31e58652f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f 2024-12-05T00:42:47,077 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f, entries=150, sequenceid=52, filesize=30.2 K 2024-12-05T00:42:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/fb0b5d2f733c4442a97f9cba481be9a7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb0b5d2f733c4442a97f9cba481be9a7 2024-12-05T00:42:47,081 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb0b5d2f733c4442a97f9cba481be9a7, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T00:42:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/565772b12cd344f682a8039a73d14f7b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/565772b12cd344f682a8039a73d14f7b 2024-12-05T00:42:47,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/565772b12cd344f682a8039a73d14f7b, entries=150, sequenceid=52, filesize=11.7 K 2024-12-05T00:42:47,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for edbe57733a3fd4ee0e06e7066e75f311 in 1653ms, sequenceid=52, compaction requested=false 2024-12-05T00:42:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T00:42:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-05T00:42:47,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T00:42:47,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4920 sec 2024-12-05T00:42:47,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.4970 sec 2024-12-05T00:42:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T00:42:47,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:47,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-05T00:42:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T00:42:47,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:47,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:47,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:47,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:42:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:47,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a0104f84e1924834aca70e9cdc6cfa35_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:47,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742177_1353 (size=14594) 2024-12-05T00:42:47,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:47,819 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a0104f84e1924834aca70e9cdc6cfa35_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a0104f84e1924834aca70e9cdc6cfa35_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:47,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a96288e44e104b6290f2a63d56a44f29, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:47,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a96288e44e104b6290f2a63d56a44f29 is 175, key is test_row_0/A:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:47,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742178_1354 (size=39549) 2024-12-05T00:42:47,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a96288e44e104b6290f2a63d56a44f29 2024-12-05T00:42:47,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/fb99f9a0d9574c47ab86060b00abb3f0 is 50, key is test_row_0/B:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742179_1355 (size=12001) 2024-12-05T00:42:47,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/fb99f9a0d9574c47ab86060b00abb3f0 2024-12-05T00:42:47,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/bd482196b5b54c9da0e5a7d0bbd81599 is 50, key is test_row_0/C:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:47,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742180_1356 (size=12001) 2024-12-05T00:42:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733359367844, max timestamp=1733359367859Num uniques 859; 2024-12-05T00:42:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733359367844, max timestamp=1733359367859Num uniques 859; 2024-12-05T00:42:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733359367844, max timestamp=1733359367859Num uniques 859; 2024-12-05T00:42:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:42:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T00:42:47,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359427901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:47,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T00:42:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359427910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359427910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:47,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359427932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359427932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:47,940 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:47,940 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359428022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359428022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359428022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T00:42:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T00:42:48,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T00:42:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359428232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359428232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359428232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/bd482196b5b54c9da0e5a7d0bbd81599 2024-12-05T00:42:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a96288e44e104b6290f2a63d56a44f29 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29 2024-12-05T00:42:48,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29, entries=200, sequenceid=62, filesize=38.6 K 2024-12-05T00:42:48,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/fb99f9a0d9574c47ab86060b00abb3f0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb99f9a0d9574c47ab86060b00abb3f0 2024-12-05T00:42:48,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb99f9a0d9574c47ab86060b00abb3f0, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T00:42:48,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/bd482196b5b54c9da0e5a7d0bbd81599 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/bd482196b5b54c9da0e5a7d0bbd81599 2024-12-05T00:42:48,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/bd482196b5b54c9da0e5a7d0bbd81599, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T00:42:48,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for edbe57733a3fd4ee0e06e7066e75f311 in 473ms, sequenceid=62, compaction requested=true 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:48,278 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:48,278 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:48,280 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:48,280 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/A is initiating minor compaction (all files) 2024-12-05T00:42:48,281 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/A in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,281 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:48,281 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=90.7 K 2024-12-05T00:42:48,281 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/B is initiating minor compaction (all files) 2024-12-05T00:42:48,281 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,281 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/B in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,281 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29] 2024-12-05T00:42:48,281 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3c8865a5c9ef47ffa0e9dae300ea4155, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb0b5d2f733c4442a97f9cba481be9a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb99f9a0d9574c47ab86060b00abb3f0] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=32.9 K 2024-12-05T00:42:48,281 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6152a928435c4f6498cd592481c4c1d7, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359363555 2024-12-05T00:42:48,281 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c8865a5c9ef47ffa0e9dae300ea4155, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359363555 2024-12-05T00:42:48,282 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting fb0b5d2f733c4442a97f9cba481be9a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733359363586 2024-12-05T00:42:48,282 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4268534fa050461bb6313fb31e58652f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733359363586 2024-12-05T00:42:48,283 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting fb99f9a0d9574c47ab86060b00abb3f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359365756 2024-12-05T00:42:48,283 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96288e44e104b6290f2a63d56a44f29, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359365756 2024-12-05T00:42:48,289 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:48,294 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:48,294 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d67e9ced5aed4e5894f7efe1cd9e7a22 is 50, key is test_row_0/B:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:48,297 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120541deeeb42b134f13a9a46410d6af6002_edbe57733a3fd4ee0e06e7066e75f311 store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:48,299 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120541deeeb42b134f13a9a46410d6af6002_edbe57733a3fd4ee0e06e7066e75f311, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:48,299 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120541deeeb42b134f13a9a46410d6af6002_edbe57733a3fd4ee0e06e7066e75f311 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:48,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742181_1357 (size=12104) 2024-12-05T00:42:48,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742182_1358 (size=4469) 2024-12-05T00:42:48,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T00:42:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,377 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T00:42:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051c76ba14d3bb48c3b58daf500321753d_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359367909/Put/seqid=0 2024-12-05T00:42:48,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742183_1359 (size=12154) 2024-12-05T00:42:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T00:42:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359428572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359428573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359428574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359428682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359428691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359428691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,717 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d67e9ced5aed4e5894f7efe1cd9e7a22 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d67e9ced5aed4e5894f7efe1cd9e7a22 2024-12-05T00:42:48,721 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/B of edbe57733a3fd4ee0e06e7066e75f311 into d67e9ced5aed4e5894f7efe1cd9e7a22(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:48,721 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:48,721 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/B, priority=13, startTime=1733359368278; duration=0sec 2024-12-05T00:42:48,721 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:48,721 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:B 2024-12-05T00:42:48,721 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:48,722 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:48,722 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/C is initiating minor compaction (all files) 2024-12-05T00:42:48,722 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/C in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:48,722 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/964c729296c44e16af9c12d0e6115aa1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/565772b12cd344f682a8039a73d14f7b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/bd482196b5b54c9da0e5a7d0bbd81599] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=32.9 K 2024-12-05T00:42:48,722 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 964c729296c44e16af9c12d0e6115aa1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359363555 2024-12-05T00:42:48,723 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 565772b12cd344f682a8039a73d14f7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733359363586 2024-12-05T00:42:48,723 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting bd482196b5b54c9da0e5a7d0bbd81599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359365756 2024-12-05T00:42:48,723 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:48,723 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/15e073c94d9d423d865c5ee007ff30ca is 175, key is test_row_0/A:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:48,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742184_1360 (size=31058) 2024-12-05T00:42:48,731 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:48,731 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/cd73e683b86e471e9138afe638d469b8 is 50, key is test_row_0/C:col10/1733359367804/Put/seqid=0 2024-12-05T00:42:48,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742185_1361 (size=12104) 2024-12-05T00:42:48,735 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/15e073c94d9d423d865c5ee007ff30ca as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/15e073c94d9d423d865c5ee007ff30ca 2024-12-05T00:42:48,739 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/A of edbe57733a3fd4ee0e06e7066e75f311 into 15e073c94d9d423d865c5ee007ff30ca(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:48,739 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:48,739 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/A, priority=13, startTime=1733359368278; duration=0sec 2024-12-05T00:42:48,739 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:48,739 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:A 2024-12-05T00:42:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:48,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051c76ba14d3bb48c3b58daf500321753d_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c76ba14d3bb48c3b58daf500321753d_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/072d015a367241858309c31f2560790c, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/072d015a367241858309c31f2560790c is 175, key is test_row_0/A:col10/1733359367909/Put/seqid=0 2024-12-05T00:42:48,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742186_1362 (size=30955) 2024-12-05T00:42:48,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359428892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T00:42:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359428902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359428911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,139 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/cd73e683b86e471e9138afe638d469b8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/cd73e683b86e471e9138afe638d469b8 2024-12-05T00:42:49,144 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/C of edbe57733a3fd4ee0e06e7066e75f311 into cd73e683b86e471e9138afe638d469b8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:49,144 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:49,144 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/C, priority=13, startTime=1733359368278; duration=0sec 2024-12-05T00:42:49,144 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:49,144 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:C 2024-12-05T00:42:49,199 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/072d015a367241858309c31f2560790c 2024-12-05T00:42:49,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/3bd76526ab49446d865d88872fb7018f is 50, key is test_row_0/B:col10/1733359367909/Put/seqid=0 2024-12-05T00:42:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359429202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359429212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359429222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742187_1363 (size=12001) 2024-12-05T00:42:49,644 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/3bd76526ab49446d865d88872fb7018f 2024-12-05T00:42:49,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/61cb08aaebc849349c7b0c2ae9c6baa9 is 50, key is test_row_0/C:col10/1733359367909/Put/seqid=0 2024-12-05T00:42:49,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742188_1364 (size=12001) 2024-12-05T00:42:49,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/61cb08aaebc849349c7b0c2ae9c6baa9 2024-12-05T00:42:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/072d015a367241858309c31f2560790c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c 2024-12-05T00:42:49,673 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c, entries=150, sequenceid=98, filesize=30.2 K 2024-12-05T00:42:49,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/3bd76526ab49446d865d88872fb7018f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3bd76526ab49446d865d88872fb7018f 2024-12-05T00:42:49,677 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3bd76526ab49446d865d88872fb7018f, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T00:42:49,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/61cb08aaebc849349c7b0c2ae9c6baa9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/61cb08aaebc849349c7b0c2ae9c6baa9 2024-12-05T00:42:49,681 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/61cb08aaebc849349c7b0c2ae9c6baa9, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T00:42:49,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for edbe57733a3fd4ee0e06e7066e75f311 in 1306ms, sequenceid=98, compaction requested=false 2024-12-05T00:42:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-05T00:42:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-05T00:42:49,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-05T00:42:49,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-12-05T00:42:49,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9290 sec 2024-12-05T00:42:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:49,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:42:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:49,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bc3de570a9564e4e9095f82197c02f92_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742189_1365 (size=14594) 2024-12-05T00:42:49,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733359369759, max timestamp=1733359369772Num uniques 859; 2024-12-05T00:42:49,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733359369759, max timestamp=1733359369772Num uniques 859; 2024-12-05T00:42:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733359369759, max timestamp=1733359369772Num uniques 859; 2024-12-05T00:42:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:42:49,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359429817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359429818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359429820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T00:42:49,912 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:49,912 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-05T00:42:49,913 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:49,914 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:49,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359429932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359429942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359429942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359429961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359429961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:49,967 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:49,968 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:50,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:50,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bc3de570a9564e4e9095f82197c02f92_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bc3de570a9564e4e9095f82197c02f92_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:50,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359430141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/757f51bd7b064848bfa1f4b00c16ce94, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:50,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/757f51bd7b064848bfa1f4b00c16ce94 is 175, key is test_row_0/A:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:50,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742190_1366 (size=39549) 2024-12-05T00:42:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359430151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359430151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:50,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359430452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359430461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359430471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:50,549 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/757f51bd7b064848bfa1f4b00c16ce94 2024-12-05T00:42:50,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d28c77f1ded84988ac6ff64217580dc2 is 50, key is test_row_0/B:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:50,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742191_1367 (size=12001) 2024-12-05T00:42:50,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d28c77f1ded84988ac6ff64217580dc2 2024-12-05T00:42:50,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/0801bf9dfaef45b3a13715d91e7d0b64 is 50, key is test_row_0/C:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359430961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742192_1368 (size=12001) 2024-12-05T00:42:50,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:50,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:50,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:50,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:50,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359430981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:50,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359430992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:51,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:51,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:51,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/0801bf9dfaef45b3a13715d91e7d0b64 2024-12-05T00:42:51,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/757f51bd7b064848bfa1f4b00c16ce94 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94 2024-12-05T00:42:51,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94, entries=200, sequenceid=112, filesize=38.6 K 2024-12-05T00:42:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d28c77f1ded84988ac6ff64217580dc2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d28c77f1ded84988ac6ff64217580dc2 2024-12-05T00:42:51,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d28c77f1ded84988ac6ff64217580dc2, entries=150, sequenceid=112, filesize=11.7 K 2024-12-05T00:42:51,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/0801bf9dfaef45b3a13715d91e7d0b64 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0801bf9dfaef45b3a13715d91e7d0b64 2024-12-05T00:42:51,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0801bf9dfaef45b3a13715d91e7d0b64, entries=150, sequenceid=112, filesize=11.7 K 2024-12-05T00:42:51,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for edbe57733a3fd4ee0e06e7066e75f311 in 1658ms, sequenceid=112, compaction requested=true 2024-12-05T00:42:51,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:51,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:51,384 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:51,384 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/A is initiating minor compaction (all files) 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/B is initiating minor compaction (all files) 2024-12-05T00:42:51,385 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/A in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,385 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/B in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,385 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/15e073c94d9d423d865c5ee007ff30ca, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=99.2 K 2024-12-05T00:42:51,385 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d67e9ced5aed4e5894f7efe1cd9e7a22, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3bd76526ab49446d865d88872fb7018f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d28c77f1ded84988ac6ff64217580dc2] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=35.3 K 2024-12-05T00:42:51,385 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/15e073c94d9d423d865c5ee007ff30ca, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94] 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d67e9ced5aed4e5894f7efe1cd9e7a22, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359365756 2024-12-05T00:42:51,385 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e073c94d9d423d865c5ee007ff30ca, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359365756 2024-12-05T00:42:51,386 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd76526ab49446d865d88872fb7018f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733359367893 2024-12-05T00:42:51,386 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072d015a367241858309c31f2560790c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733359367893 2024-12-05T00:42:51,386 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d28c77f1ded84988ac6ff64217580dc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733359368562 2024-12-05T00:42:51,386 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 757f51bd7b064848bfa1f4b00c16ce94, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733359368562 2024-12-05T00:42:51,390 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:51,391 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:51,391 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/8090a038f059499d931087312a56bd13 is 50, key is test_row_0/B:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:51,392 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120556b96a47cf424408ad2375ce10dc09d1_edbe57733a3fd4ee0e06e7066e75f311 store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:51,394 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120556b96a47cf424408ad2375ce10dc09d1_edbe57733a3fd4ee0e06e7066e75f311, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:51,394 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120556b96a47cf424408ad2375ce10dc09d1_edbe57733a3fd4ee0e06e7066e75f311 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:51,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742193_1369 (size=12207) 2024-12-05T00:42:51,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742194_1370 (size=4469) 2024-12-05T00:42:51,399 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#A#compaction#316 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:51,399 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/c9aaf0d9270945ef911c6d457c6af984 is 175, key is test_row_0/A:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:51,402 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/8090a038f059499d931087312a56bd13 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/8090a038f059499d931087312a56bd13 2024-12-05T00:42:51,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742195_1371 (size=31161) 2024-12-05T00:42:51,409 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/B of edbe57733a3fd4ee0e06e7066e75f311 into 8090a038f059499d931087312a56bd13(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:51,409 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:51,409 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/B, priority=13, startTime=1733359371384; duration=0sec 2024-12-05T00:42:51,410 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:51,410 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:B 2024-12-05T00:42:51,410 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:51,410 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:51,410 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/C is initiating minor compaction (all files) 2024-12-05T00:42:51,411 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/C in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,411 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/cd73e683b86e471e9138afe638d469b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/61cb08aaebc849349c7b0c2ae9c6baa9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0801bf9dfaef45b3a13715d91e7d0b64] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=35.3 K 2024-12-05T00:42:51,411 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting cd73e683b86e471e9138afe638d469b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359365756 2024-12-05T00:42:51,411 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/c9aaf0d9270945ef911c6d457c6af984 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/c9aaf0d9270945ef911c6d457c6af984 2024-12-05T00:42:51,411 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 61cb08aaebc849349c7b0c2ae9c6baa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733359367893 2024-12-05T00:42:51,411 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0801bf9dfaef45b3a13715d91e7d0b64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733359368562 2024-12-05T00:42:51,415 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/A of edbe57733a3fd4ee0e06e7066e75f311 into c9aaf0d9270945ef911c6d457c6af984(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:51,415 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:51,415 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/A, priority=13, startTime=1733359371383; duration=0sec 2024-12-05T00:42:51,415 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:51,415 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:A 2024-12-05T00:42:51,416 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#C#compaction#317 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:51,417 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/2000dd124e1841f7862ecce2618ada24 is 50, key is test_row_0/C:col10/1733359369724/Put/seqid=0 2024-12-05T00:42:51,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742196_1372 (size=12207) 2024-12-05T00:42:51,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:51,439 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:42:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205317b9425d637487b9b25b6f9898a7517_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359369816/Put/seqid=0 2024-12-05T00:42:51,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742197_1373 (size=12304) 2024-12-05T00:42:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:51,462 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205317b9425d637487b9b25b6f9898a7517_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205317b9425d637487b9b25b6f9898a7517_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/1b19ab98d01b489e982f594907f9c2ed, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/1b19ab98d01b489e982f594907f9c2ed is 175, key is test_row_0/A:col10/1733359369816/Put/seqid=0 2024-12-05T00:42:51,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742198_1374 (size=31105) 2024-12-05T00:42:51,823 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/2000dd124e1841f7862ecce2618ada24 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/2000dd124e1841f7862ecce2618ada24 2024-12-05T00:42:51,827 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/C of edbe57733a3fd4ee0e06e7066e75f311 into 2000dd124e1841f7862ecce2618ada24(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:51,827 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:51,827 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/C, priority=13, startTime=1733359371384; duration=0sec 2024-12-05T00:42:51,827 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:51,827 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:C 2024-12-05T00:42:51,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/1b19ab98d01b489e982f594907f9c2ed 2024-12-05T00:42:51,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/49efa8fc9a394887addc056eb79a4620 is 50, key is test_row_0/B:col10/1733359369816/Put/seqid=0 2024-12-05T00:42:51,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742199_1375 (size=12151) 2024-12-05T00:42:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:51,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:52,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359432019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359432020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359432024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359432132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359432132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359432141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,288 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/49efa8fc9a394887addc056eb79a4620 2024-12-05T00:42:52,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/458cf94a50d345f1aa62e728f4a97662 is 50, key is test_row_0/C:col10/1733359369816/Put/seqid=0 2024-12-05T00:42:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742200_1376 (size=12151) 2024-12-05T00:42:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359432341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359432341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359432351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359432652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359432652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359432662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:52,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:52,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:52,701 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/458cf94a50d345f1aa62e728f4a97662 2024-12-05T00:42:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/1b19ab98d01b489e982f594907f9c2ed as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed 2024-12-05T00:42:52,708 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed, entries=150, sequenceid=148, filesize=30.4 K 2024-12-05T00:42:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/49efa8fc9a394887addc056eb79a4620 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/49efa8fc9a394887addc056eb79a4620 2024-12-05T00:42:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,711 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/49efa8fc9a394887addc056eb79a4620, entries=150, sequenceid=148, filesize=11.9 K 2024-12-05T00:42:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/458cf94a50d345f1aa62e728f4a97662 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/458cf94a50d345f1aa62e728f4a97662 2024-12-05T00:42:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/458cf94a50d345f1aa62e728f4a97662, entries=150, sequenceid=148, filesize=11.9 K 2024-12-05T00:42:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for edbe57733a3fd4ee0e06e7066e75f311 in 1277ms, sequenceid=148, compaction requested=false 2024-12-05T00:42:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-05T00:42:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-05T00:42:52,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8030 sec 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.8080 sec 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:42:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205034809dbb8414130b1b9d5fef219f98c_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742201_1377 (size=12304) 2024-12-05T00:42:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359433242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359433243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359433254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359433361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359433362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359433372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359433571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359433571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359433581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:53,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205034809dbb8414130b1b9d5fef219f98c_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205034809dbb8414130b1b9d5fef219f98c_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:53,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/f925420bf3a743ea8bf9fc1958c2e918, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:53,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/f925420bf3a743ea8bf9fc1958c2e918 is 175, key is test_row_0/A:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:53,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742202_1378 (size=31105) 2024-12-05T00:42:53,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359433882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359433882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359433892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50516 deadline: 1733359433982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,985 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50554 deadline: 1733359433991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:53,997 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:54,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/f925420bf3a743ea8bf9fc1958c2e918 2024-12-05T00:42:54,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d808428d64064f6b98c1e75ca2b79543 is 50, key is test_row_0/B:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:54,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742203_1379 (size=12151) 2024-12-05T00:42:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T00:42:54,072 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:54,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-05T00:42:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:54,074 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:54,075 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:54,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:54,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T00:42:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T00:42:54,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:54,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359434391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359434402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:54,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d808428d64064f6b98c1e75ca2b79543 2024-12-05T00:42:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359434412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:54,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/d2b9c26e68cc45f6ae7a64d4caf9f58f is 50, key is test_row_0/C:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742204_1380 (size=12151) 2024-12-05T00:42:54,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T00:42:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T00:42:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:54,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/d2b9c26e68cc45f6ae7a64d4caf9f58f 2024-12-05T00:42:54,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/f925420bf3a743ea8bf9fc1958c2e918 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918 2024-12-05T00:42:54,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918, entries=150, sequenceid=161, filesize=30.4 K 2024-12-05T00:42:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/d808428d64064f6b98c1e75ca2b79543 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d808428d64064f6b98c1e75ca2b79543 2024-12-05T00:42:54,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d808428d64064f6b98c1e75ca2b79543, entries=150, sequenceid=161, filesize=11.9 K 2024-12-05T00:42:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/d2b9c26e68cc45f6ae7a64d4caf9f58f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d2b9c26e68cc45f6ae7a64d4caf9f58f 2024-12-05T00:42:54,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T00:42:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d2b9c26e68cc45f6ae7a64d4caf9f58f, entries=150, sequenceid=161, filesize=11.9 K 2024-12-05T00:42:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:42:54,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for edbe57733a3fd4ee0e06e7066e75f311 in 1652ms, sequenceid=161, compaction requested=true 2024-12-05T00:42:54,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:54,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:54,838 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:54,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:54,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:54,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:54,839 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:54,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:54,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:54,839 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:54,839 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/A is initiating minor compaction (all files) 2024-12-05T00:42:54,839 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/A in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,839 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/c9aaf0d9270945ef911c6d457c6af984, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=91.2 K 2024-12-05T00:42:54,840 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,840 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/c9aaf0d9270945ef911c6d457c6af984, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918] 2024-12-05T00:42:54,840 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:54,840 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/B is initiating minor compaction (all files) 2024-12-05T00:42:54,840 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/B in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,840 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/8090a038f059499d931087312a56bd13, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/49efa8fc9a394887addc056eb79a4620, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d808428d64064f6b98c1e75ca2b79543] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=35.7 K 2024-12-05T00:42:54,841 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8090a038f059499d931087312a56bd13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733359368562 2024-12-05T00:42:54,841 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9aaf0d9270945ef911c6d457c6af984, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733359368562 2024-12-05T00:42:54,841 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 49efa8fc9a394887addc056eb79a4620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733359369815 2024-12-05T00:42:54,841 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b19ab98d01b489e982f594907f9c2ed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733359369815 2024-12-05T00:42:54,841 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d808428d64064f6b98c1e75ca2b79543, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733359372010 2024-12-05T00:42:54,841 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f925420bf3a743ea8bf9fc1958c2e918, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733359372010 2024-12-05T00:42:54,857 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:54,858 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:54,859 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052fcb947bdd19449c8a33d2f067db28ca_edbe57733a3fd4ee0e06e7066e75f311 store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:54,859 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/bd3b11dd48e346199e81a7b62448b5d4 is 50, key is test_row_0/B:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:54,860 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052fcb947bdd19449c8a33d2f067db28ca_edbe57733a3fd4ee0e06e7066e75f311, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:54,860 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052fcb947bdd19449c8a33d2f067db28ca_edbe57733a3fd4ee0e06e7066e75f311 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:54,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742206_1382 (size=4469) 2024-12-05T00:42:54,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742205_1381 (size=12459) 2024-12-05T00:42:54,872 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/bd3b11dd48e346199e81a7b62448b5d4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/bd3b11dd48e346199e81a7b62448b5d4 2024-12-05T00:42:54,875 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/B of edbe57733a3fd4ee0e06e7066e75f311 into bd3b11dd48e346199e81a7b62448b5d4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:54,875 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:54,875 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/B, priority=13, startTime=1733359374838; duration=0sec 2024-12-05T00:42:54,875 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:54,875 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:B 2024-12-05T00:42:54,876 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:54,876 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:54,876 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/C is initiating minor compaction (all files) 2024-12-05T00:42:54,876 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/C in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,876 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/2000dd124e1841f7862ecce2618ada24, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/458cf94a50d345f1aa62e728f4a97662, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d2b9c26e68cc45f6ae7a64d4caf9f58f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=35.7 K 2024-12-05T00:42:54,877 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2000dd124e1841f7862ecce2618ada24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733359368562 2024-12-05T00:42:54,877 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 458cf94a50d345f1aa62e728f4a97662, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733359369815 2024-12-05T00:42:54,877 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b9c26e68cc45f6ae7a64d4caf9f58f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733359372010 2024-12-05T00:42:54,884 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:54,884 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/583c789d857049d99cea35e05ce22c30 is 50, key is test_row_0/C:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:54,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742207_1383 (size=12459) 2024-12-05T00:42:54,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-05T00:42:54,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:54,989 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T00:42:54,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:54,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:54,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:54,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:54,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205df382c531d994f2c9e13906da89f172b_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359373242/Put/seqid=0 2024-12-05T00:42:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742208_1384 (size=12304) 2024-12-05T00:42:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:55,268 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:55,268 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/bdd1b72a1a034f538c9bf6fa7900eead is 175, key is test_row_0/A:col10/1733359373185/Put/seqid=0 2024-12-05T00:42:55,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742209_1385 (size=31413) 2024-12-05T00:42:55,277 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/bdd1b72a1a034f538c9bf6fa7900eead as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/bdd1b72a1a034f538c9bf6fa7900eead 2024-12-05T00:42:55,281 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/A of edbe57733a3fd4ee0e06e7066e75f311 into bdd1b72a1a034f538c9bf6fa7900eead(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:55,281 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:55,281 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/A, priority=13, startTime=1733359374838; duration=0sec 2024-12-05T00:42:55,281 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:55,281 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:A 2024-12-05T00:42:55,294 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/583c789d857049d99cea35e05ce22c30 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/583c789d857049d99cea35e05ce22c30 2024-12-05T00:42:55,298 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/C of edbe57733a3fd4ee0e06e7066e75f311 into 583c789d857049d99cea35e05ce22c30(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:55,298 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:55,298 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/C, priority=13, startTime=1733359374839; duration=0sec 2024-12-05T00:42:55,298 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:55,298 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:C 2024-12-05T00:42:55,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:55,402 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205df382c531d994f2c9e13906da89f172b_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205df382c531d994f2c9e13906da89f172b_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/851ddb19f062447bb61ffcd4a86a98b1, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:55,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/851ddb19f062447bb61ffcd4a86a98b1 is 175, key is test_row_0/A:col10/1733359373242/Put/seqid=0 2024-12-05T00:42:55,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742210_1386 (size=31105) 2024-12-05T00:42:55,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359435432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,438 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359435435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359435436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359435552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359435552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359435761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359435761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:55,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/851ddb19f062447bb61ffcd4a86a98b1 2024-12-05T00:42:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/2456892d1c6c4ffaac0a3ce97868d653 is 50, key is test_row_0/B:col10/1733359373242/Put/seqid=0 2024-12-05T00:42:55,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742211_1387 (size=12151) 2024-12-05T00:42:56,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359436072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:56,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359436072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:56,229 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/2456892d1c6c4ffaac0a3ce97868d653 2024-12-05T00:42:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/710f85766f2f4da58a57b60c43fd1feb is 50, key is test_row_0/C:col10/1733359373242/Put/seqid=0 2024-12-05T00:42:56,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742212_1388 (size=12151) 2024-12-05T00:42:56,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359436592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:56,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359436592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:56,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/710f85766f2f4da58a57b60c43fd1feb 2024-12-05T00:42:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/851ddb19f062447bb61ffcd4a86a98b1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1 2024-12-05T00:42:56,645 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1, entries=150, sequenceid=182, filesize=30.4 K 2024-12-05T00:42:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/2456892d1c6c4ffaac0a3ce97868d653 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/2456892d1c6c4ffaac0a3ce97868d653 2024-12-05T00:42:56,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/2456892d1c6c4ffaac0a3ce97868d653, entries=150, sequenceid=182, filesize=11.9 K 2024-12-05T00:42:56,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/710f85766f2f4da58a57b60c43fd1feb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/710f85766f2f4da58a57b60c43fd1feb 2024-12-05T00:42:56,655 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/710f85766f2f4da58a57b60c43fd1feb, entries=150, sequenceid=182, filesize=11.9 K 2024-12-05T00:42:56,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for edbe57733a3fd4ee0e06e7066e75f311 in 1667ms, sequenceid=182, compaction requested=false 2024-12-05T00:42:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-05T00:42:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-05T00:42:56,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-05T00:42:56,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5820 sec 2024-12-05T00:42:56,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.5860 sec 2024-12-05T00:42:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:57,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:57,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f46b8614082145249aa14724f09373e2_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:57,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742213_1389 (size=14794) 2024-12-05T00:42:57,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:57,478 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f46b8614082145249aa14724f09373e2_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f46b8614082145249aa14724f09373e2_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:57,479 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a466aa81ee4e4e97a84779c9fa0af8f7, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:57,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a466aa81ee4e4e97a84779c9fa0af8f7 is 175, key is test_row_0/A:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:57,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742214_1390 (size=39749) 2024-12-05T00:42:57,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a466aa81ee4e4e97a84779c9fa0af8f7 2024-12-05T00:42:57,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/85b8c2ec0ac444a5bcc06759e5bb8387 is 50, key is test_row_0/B:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742215_1391 (size=12151) 2024-12-05T00:42:57,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359437557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359437601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:57,607 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:57,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359437612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:57,621 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:57,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:57,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359437661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359437872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:57,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/85b8c2ec0ac444a5bcc06759e5bb8387 2024-12-05T00:42:57,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/6b3ad713aee24b11ac0cde7d9be0c4cb is 50, key is test_row_0/C:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742216_1392 (size=12151) 2024-12-05T00:42:58,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359438182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T00:42:58,242 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:42:58,242 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:42:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-05T00:42:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T00:42:58,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:42:58,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:42:58,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:42:58,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/6b3ad713aee24b11ac0cde7d9be0c4cb 2024-12-05T00:42:58,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/a466aa81ee4e4e97a84779c9fa0af8f7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7 2024-12-05T00:42:58,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7, entries=200, sequenceid=197, filesize=38.8 K 2024-12-05T00:42:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/85b8c2ec0ac444a5bcc06759e5bb8387 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/85b8c2ec0ac444a5bcc06759e5bb8387 2024-12-05T00:42:58,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/85b8c2ec0ac444a5bcc06759e5bb8387, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T00:42:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/6b3ad713aee24b11ac0cde7d9be0c4cb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6b3ad713aee24b11ac0cde7d9be0c4cb 2024-12-05T00:42:58,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6b3ad713aee24b11ac0cde7d9be0c4cb, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T00:42:58,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for edbe57733a3fd4ee0e06e7066e75f311 in 863ms, sequenceid=197, compaction requested=true 2024-12-05T00:42:58,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:58,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:42:58,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:58,328 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:58,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:42:58,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:58,328 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:58,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:42:58,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:58,329 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:58,329 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:58,329 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/A is initiating minor compaction (all files) 2024-12-05T00:42:58,329 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/B is initiating minor compaction (all files) 2024-12-05T00:42:58,329 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/A in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:58,329 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/B in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:58,329 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/bd3b11dd48e346199e81a7b62448b5d4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/2456892d1c6c4ffaac0a3ce97868d653, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/85b8c2ec0ac444a5bcc06759e5bb8387] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=35.9 K 2024-12-05T00:42:58,329 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/bdd1b72a1a034f538c9bf6fa7900eead, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=99.9 K 2024-12-05T00:42:58,329 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/bdd1b72a1a034f538c9bf6fa7900eead, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7] 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3b11dd48e346199e81a7b62448b5d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733359372010 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdd1b72a1a034f538c9bf6fa7900eead, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733359372010 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2456892d1c6c4ffaac0a3ce97868d653, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733359373240 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851ddb19f062447bb61ffcd4a86a98b1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733359373240 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 85b8c2ec0ac444a5bcc06759e5bb8387, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733359375427 2024-12-05T00:42:58,330 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a466aa81ee4e4e97a84779c9fa0af8f7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733359375427 2024-12-05T00:42:58,334 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:58,336 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412057caf32d2c8da4af98c58ca82df073212_edbe57733a3fd4ee0e06e7066e75f311 store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:58,336 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:58,336 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/ef4aaaf7d4c24f42b965226f38b19a9d is 50, key is test_row_0/B:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:58,337 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412057caf32d2c8da4af98c58ca82df073212_edbe57733a3fd4ee0e06e7066e75f311, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:58,338 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057caf32d2c8da4af98c58ca82df073212_edbe57733a3fd4ee0e06e7066e75f311 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742218_1394 (size=12561) 2024-12-05T00:42:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742217_1393 (size=4469) 2024-12-05T00:42:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T00:42:58,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-05T00:42:58,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:58,396 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:42:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:42:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:42:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:42:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:42:58,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fbd6ee8307a0445e8814290df842fd40_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359377550/Put/seqid=0 2024-12-05T00:42:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742219_1395 (size=12304) 2024-12-05T00:42:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T00:42:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:58,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:42:58,746 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/ef4aaaf7d4c24f42b965226f38b19a9d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/ef4aaaf7d4c24f42b965226f38b19a9d 2024-12-05T00:42:58,747 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:58,747 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/8e1ed171408943a39a65dfd8df091398 is 175, key is test_row_0/A:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:58,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742220_1396 (size=31515) 2024-12-05T00:42:58,751 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/B of edbe57733a3fd4ee0e06e7066e75f311 into ef4aaaf7d4c24f42b965226f38b19a9d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:58,751 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:58,751 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/B, priority=13, startTime=1733359378328; duration=0sec 2024-12-05T00:42:58,751 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:42:58,751 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:B 2024-12-05T00:42:58,751 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:42:58,753 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:42:58,753 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): edbe57733a3fd4ee0e06e7066e75f311/C is initiating minor compaction (all files) 2024-12-05T00:42:58,753 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edbe57733a3fd4ee0e06e7066e75f311/C in TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:42:58,753 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/583c789d857049d99cea35e05ce22c30, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/710f85766f2f4da58a57b60c43fd1feb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6b3ad713aee24b11ac0cde7d9be0c4cb] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp, totalSize=35.9 K 2024-12-05T00:42:58,753 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 583c789d857049d99cea35e05ce22c30, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733359372010 2024-12-05T00:42:58,753 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/8e1ed171408943a39a65dfd8df091398 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/8e1ed171408943a39a65dfd8df091398 2024-12-05T00:42:58,754 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 710f85766f2f4da58a57b60c43fd1feb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733359373240 2024-12-05T00:42:58,754 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3ad713aee24b11ac0cde7d9be0c4cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733359375427 2024-12-05T00:42:58,758 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/A of edbe57733a3fd4ee0e06e7066e75f311 into 8e1ed171408943a39a65dfd8df091398(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:58,758 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:58,758 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/A, priority=13, startTime=1733359378328; duration=0sec 2024-12-05T00:42:58,758 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:58,758 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:A 2024-12-05T00:42:58,761 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edbe57733a3fd4ee0e06e7066e75f311#C#compaction#336 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:42:58,762 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/847a7d558a9948a098a6765c30d9659d is 50, key is test_row_0/C:col10/1733359375427/Put/seqid=0 2024-12-05T00:42:58,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359438756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:58,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:58,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742221_1397 (size=12561) 2024-12-05T00:42:58,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:42:58,808 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fbd6ee8307a0445e8814290df842fd40_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fbd6ee8307a0445e8814290df842fd40_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:42:58,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/3644772e454047e1b4a1da5a091a2d0b, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:42:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/3644772e454047e1b4a1da5a091a2d0b is 175, key is test_row_0/A:col10/1733359377550/Put/seqid=0 2024-12-05T00:42:58,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742222_1398 (size=31105) 2024-12-05T00:42:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T00:42:58,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359438872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359439081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:59,171 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/847a7d558a9948a098a6765c30d9659d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/847a7d558a9948a098a6765c30d9659d 2024-12-05T00:42:59,174 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edbe57733a3fd4ee0e06e7066e75f311/C of edbe57733a3fd4ee0e06e7066e75f311 into 847a7d558a9948a098a6765c30d9659d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:42:59,174 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:42:59,174 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., storeName=edbe57733a3fd4ee0e06e7066e75f311/C, priority=13, startTime=1733359378328; duration=0sec 2024-12-05T00:42:59,174 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:42:59,174 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:C 2024-12-05T00:42:59,213 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/3644772e454047e1b4a1da5a091a2d0b 2024-12-05T00:42:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/34d40dcccf1d4af7987126041a267de8 is 50, key is test_row_0/B:col10/1733359377550/Put/seqid=0 2024-12-05T00:42:59,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742223_1399 (size=12151) 2024-12-05T00:42:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T00:42:59,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359439392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:59,623 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/34d40dcccf1d4af7987126041a267de8 2024-12-05T00:42:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50518 deadline: 1733359439622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:59,628 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/59ae71624e034511991931e9a5e79635 is 50, key is test_row_0/C:col10/1733359377550/Put/seqid=0 2024-12-05T00:42:59,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742224_1400 (size=12151) 2024-12-05T00:42:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50538 deadline: 1733359439632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:59,637 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in edbe57733a3fd4ee0e06e7066e75f311 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:42:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:42:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:42:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359439901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:00,032 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/59ae71624e034511991931e9a5e79635 2024-12-05T00:43:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/3644772e454047e1b4a1da5a091a2d0b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/3644772e454047e1b4a1da5a091a2d0b 2024-12-05T00:43:00,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/3644772e454047e1b4a1da5a091a2d0b, entries=150, sequenceid=216, filesize=30.4 K 2024-12-05T00:43:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/34d40dcccf1d4af7987126041a267de8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/34d40dcccf1d4af7987126041a267de8 2024-12-05T00:43:00,042 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/34d40dcccf1d4af7987126041a267de8, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T00:43:00,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/59ae71624e034511991931e9a5e79635 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/59ae71624e034511991931e9a5e79635 2024-12-05T00:43:00,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/59ae71624e034511991931e9a5e79635, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T00:43:00,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for edbe57733a3fd4ee0e06e7066e75f311 in 1649ms, sequenceid=216, compaction requested=false 2024-12-05T00:43:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:43:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-05T00:43:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-05T00:43:00,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-05T00:43:00,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-12-05T00:43:00,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8070 sec 2024-12-05T00:43:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T00:43:00,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:00,392 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-05T00:43:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T00:43:00,394 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:00,394 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:00,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T00:43:00,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:00,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:43:00,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fc6b38a1cadc4751a89ca865f81b0b88_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359378728/Put/seqid=0 2024-12-05T00:43:00,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742225_1401 (size=12304) 2024-12-05T00:43:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T00:43:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:00,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:43:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:00,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fc6b38a1cadc4751a89ca865f81b0b88_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fc6b38a1cadc4751a89ca865f81b0b88_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:00,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/0a7d3cb4e2c14c3da56f2754b5908447, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:43:00,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/0a7d3cb4e2c14c3da56f2754b5908447 is 175, key is test_row_0/A:col10/1733359378728/Put/seqid=0 2024-12-05T00:43:00,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742226_1402 (size=31105) 2024-12-05T00:43:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359441011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T00:43:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359441121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359441332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:01,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/0a7d3cb4e2c14c3da56f2754b5908447 2024-12-05T00:43:01,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/b5ca6297d0564b4894f3ed742e25047e is 50, key is test_row_0/B:col10/1733359378728/Put/seqid=0 2024-12-05T00:43:01,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742227_1403 (size=12151) 2024-12-05T00:43:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T00:43:01,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359441641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:01,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/b5ca6297d0564b4894f3ed742e25047e 2024-12-05T00:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/0a9fca99ba96467282a28d010408d761 is 50, key is test_row_0/C:col10/1733359378728/Put/seqid=0 2024-12-05T00:43:01,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742228_1404 (size=12151) 2024-12-05T00:43:02,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359442152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:02,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/0a9fca99ba96467282a28d010408d761 2024-12-05T00:43:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/0a7d3cb4e2c14c3da56f2754b5908447 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/0a7d3cb4e2c14c3da56f2754b5908447 2024-12-05T00:43:02,187 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/0a7d3cb4e2c14c3da56f2754b5908447, entries=150, sequenceid=232, filesize=30.4 K 2024-12-05T00:43:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/b5ca6297d0564b4894f3ed742e25047e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b5ca6297d0564b4894f3ed742e25047e 2024-12-05T00:43:02,191 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b5ca6297d0564b4894f3ed742e25047e, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T00:43:02,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/0a9fca99ba96467282a28d010408d761 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0a9fca99ba96467282a28d010408d761 2024-12-05T00:43:02,194 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0a9fca99ba96467282a28d010408d761, entries=150, sequenceid=232, filesize=11.9 K 2024-12-05T00:43:02,195 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for edbe57733a3fd4ee0e06e7066e75f311 in 1648ms, sequenceid=232, compaction requested=true 2024-12-05T00:43:02,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:43:02,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:02,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-05T00:43:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-05T00:43:02,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-05T00:43:02,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-12-05T00:43:02,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8060 sec 2024-12-05T00:43:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T00:43:02,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:02,542 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:02,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-05T00:43:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T00:43:02,544 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:02,545 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:02,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T00:43:02,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:02,697 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:43:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:02,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051e7d7f1fde7041cda1709fba4e255bc4_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359381001/Put/seqid=0 2024-12-05T00:43:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742229_1405 (size=12304) 2024-12-05T00:43:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:02,713 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051e7d7f1fde7041cda1709fba4e255bc4_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051e7d7f1fde7041cda1709fba4e255bc4_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:02,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/18c6d34939804d338c7fd6848ae486e6, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:43:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/18c6d34939804d338c7fd6848ae486e6 is 175, key is test_row_0/A:col10/1733359381001/Put/seqid=0 2024-12-05T00:43:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742230_1406 (size=31105) 2024-12-05T00:43:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T00:43:03,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/18c6d34939804d338c7fd6848ae486e6 2024-12-05T00:43:03,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/28c10e76027c4c73a72317c5ce393359 is 50, key is test_row_0/B:col10/1733359381001/Put/seqid=0 2024-12-05T00:43:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742231_1407 (size=12151) 2024-12-05T00:43:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T00:43:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. as already flushing 2024-12-05T00:43:03,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359443229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:03,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1733359443352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:03,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/28c10e76027c4c73a72317c5ce393359 2024-12-05T00:43:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/d97bbc23f0394d13b24fb0fcbac8d356 is 50, key is test_row_0/C:col10/1733359381001/Put/seqid=0 2024-12-05T00:43:03,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742232_1408 (size=12151) 2024-12-05T00:43:03,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/d97bbc23f0394d13b24fb0fcbac8d356 2024-12-05T00:43:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/18c6d34939804d338c7fd6848ae486e6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/18c6d34939804d338c7fd6848ae486e6 2024-12-05T00:43:03,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/18c6d34939804d338c7fd6848ae486e6, entries=150, sequenceid=251, filesize=30.4 K 2024-12-05T00:43:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/28c10e76027c4c73a72317c5ce393359 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/28c10e76027c4c73a72317c5ce393359 2024-12-05T00:43:03,564 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/28c10e76027c4c73a72317c5ce393359, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T00:43:03,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/d97bbc23f0394d13b24fb0fcbac8d356 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d97bbc23f0394d13b24fb0fcbac8d356 2024-12-05T00:43:03,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d97bbc23f0394d13b24fb0fcbac8d356, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T00:43:03,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for edbe57733a3fd4ee0e06e7066e75f311 in 871ms, sequenceid=251, compaction requested=true 2024-12-05T00:43:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:43:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-05T00:43:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-05T00:43:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:03,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T00:43:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:43:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:43:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:43:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:03,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-05T00:43:03,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-12-05T00:43:03,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0310 sec 2024-12-05T00:43:03,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205934bbadede5b498db33205fd0eb098e5_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359383228/Put/seqid=0 2024-12-05T00:43:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742233_1409 (size=14994) 2024-12-05T00:43:03,608 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-12-05T00:43:03,608 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,608 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,608 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-12-05T00:43:03,608 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,608 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,608 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,608 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,609 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-12-05T00:43:03,609 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,609 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,609 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,611 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-12-05T00:43:03,611 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,611 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,611 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,616 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-12-05T00:43:03,616 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,616 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,616 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,618 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-12-05T00:43:03,618 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,618 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,618 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,643 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-12-05T00:43:03,643 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,643 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,643 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T00:43:03,682 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:03,684 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-12-05T00:43:03,685 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:03,685 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,685 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:03,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:03,992 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:04,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205934bbadede5b498db33205fd0eb098e5_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205934bbadede5b498db33205fd0eb098e5_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:04,003 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/e043e31b67ca4831b7aacc727c6ee7ea, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:43:04,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/e043e31b67ca4831b7aacc727c6ee7ea is 175, key is test_row_0/A:col10/1733359383228/Put/seqid=0 2024-12-05T00:43:04,004 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-12-05T00:43:04,005 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:04,005 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:04,005 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:04,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742234_1410 (size=39949) 2024-12-05T00:43:04,106 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-12-05T00:43:04,107 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:04,107 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:04,107 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:04,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 4 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1663 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4989 rows 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1665 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4995 rows 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1654 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4962 rows 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1666 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4998 rows 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1658 2024-12-05T00:43:04,108 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4974 rows 2024-12-05T00:43:04,108 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:43:04,109 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:43:04,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:04,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:04,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:04,110 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T00:43:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:04,114 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359384114"}]},"ts":"1733359384114"} 2024-12-05T00:43:04,116 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T00:43:04,116 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T00:43:04,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:43:04,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, UNASSIGN}] 2024-12-05T00:43:04,119 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, UNASSIGN 2024-12-05T00:43:04,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:04,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, UNASSIGN because future has completed 2024-12-05T00:43:04,121 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:43:04,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:43:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:04,275 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:04,275 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:43:04,275 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing edbe57733a3fd4ee0e06e7066e75f311, disabling compactions & flushes 2024-12-05T00:43:04,276 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:04,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/e043e31b67ca4831b7aacc727c6ee7ea 2024-12-05T00:43:04,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/b001ddb6b77e43bf8c0311de09772278 is 50, key is test_row_0/B:col10/1733359383228/Put/seqid=0 2024-12-05T00:43:04,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742235_1411 (size=12301) 2024-12-05T00:43:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:04,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/b001ddb6b77e43bf8c0311de09772278 2024-12-05T00:43:04,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/6e77167f68bd41ee863aafa7a95854fe is 50, key is test_row_0/C:col10/1733359383228/Put/seqid=0 2024-12-05T00:43:04,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742236_1412 (size=12301) 2024-12-05T00:43:05,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/6e77167f68bd41ee863aafa7a95854fe 2024-12-05T00:43:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:05,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/e043e31b67ca4831b7aacc727c6ee7ea as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/e043e31b67ca4831b7aacc727c6ee7ea 2024-12-05T00:43:05,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/e043e31b67ca4831b7aacc727c6ee7ea, entries=200, sequenceid=265, filesize=39.0 K 2024-12-05T00:43:05,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/b001ddb6b77e43bf8c0311de09772278 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b001ddb6b77e43bf8c0311de09772278 2024-12-05T00:43:05,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b001ddb6b77e43bf8c0311de09772278, entries=150, sequenceid=265, filesize=12.0 K 2024-12-05T00:43:05,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/6e77167f68bd41ee863aafa7a95854fe as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6e77167f68bd41ee863aafa7a95854fe 2024-12-05T00:43:05,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6e77167f68bd41ee863aafa7a95854fe, entries=150, sequenceid=265, filesize=12.0 K 2024-12-05T00:43:05,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=67.09 KB/68700 for edbe57733a3fd4ee0e06e7066e75f311 in 1702ms, sequenceid=265, compaction requested=true 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edbe57733a3fd4ee0e06e7066e75f311: 2024-12-05T00:43:05,274 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. after waiting 0 ms 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:05,274 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing edbe57733a3fd4ee0e06e7066e75f311 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:43:05,274 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. because compaction request was cancelled 2024-12-05T00:43:05,274 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. because compaction request was cancelled 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edbe57733a3fd4ee0e06e7066e75f311:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:05,274 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:A 2024-12-05T00:43:05,274 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:B 2024-12-05T00:43:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:05,274 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. because compaction request was cancelled 2024-12-05T00:43:05,274 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edbe57733a3fd4ee0e06e7066e75f311:C 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=A 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=B 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edbe57733a3fd4ee0e06e7066e75f311, store=C 2024-12-05T00:43:05,274 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:05,280 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120573f9d5b845be44f19e41d8a2df024e5e_edbe57733a3fd4ee0e06e7066e75f311 is 50, key is test_row_0/A:col10/1733359384103/Put/seqid=0 2024-12-05T00:43:05,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742237_1413 (size=12454) 2024-12-05T00:43:05,591 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T00:43:05,684 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:05,690 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120573f9d5b845be44f19e41d8a2df024e5e_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120573f9d5b845be44f19e41d8a2df024e5e_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:05,691 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/d515c5a2aa0849d3a51fd6806285a66d, store: [table=TestAcidGuarantees family=A region=edbe57733a3fd4ee0e06e7066e75f311] 2024-12-05T00:43:05,692 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/d515c5a2aa0849d3a51fd6806285a66d is 175, key is test_row_0/A:col10/1733359384103/Put/seqid=0 2024-12-05T00:43:05,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742238_1414 (size=31255) 2024-12-05T00:43:06,098 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/d515c5a2aa0849d3a51fd6806285a66d 2024-12-05T00:43:06,107 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/97e379fa3e1f458c9c0551016f50b153 is 50, key is test_row_0/B:col10/1733359384103/Put/seqid=0 2024-12-05T00:43:06,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742239_1415 (size=12301) 2024-12-05T00:43:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:06,512 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/97e379fa3e1f458c9c0551016f50b153 2024-12-05T00:43:06,523 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/ff33a30947b741dfa240668612661bd7 is 50, key is test_row_0/C:col10/1733359384103/Put/seqid=0 2024-12-05T00:43:06,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742240_1416 (size=12301) 2024-12-05T00:43:06,927 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/ff33a30947b741dfa240668612661bd7 2024-12-05T00:43:06,935 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/A/d515c5a2aa0849d3a51fd6806285a66d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/d515c5a2aa0849d3a51fd6806285a66d 2024-12-05T00:43:06,940 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/d515c5a2aa0849d3a51fd6806285a66d, entries=150, sequenceid=278, filesize=30.5 K 2024-12-05T00:43:06,941 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/B/97e379fa3e1f458c9c0551016f50b153 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/97e379fa3e1f458c9c0551016f50b153 2024-12-05T00:43:06,946 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/97e379fa3e1f458c9c0551016f50b153, entries=150, sequenceid=278, filesize=12.0 K 2024-12-05T00:43:06,946 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/.tmp/C/ff33a30947b741dfa240668612661bd7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/ff33a30947b741dfa240668612661bd7 2024-12-05T00:43:06,949 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/ff33a30947b741dfa240668612661bd7, entries=150, sequenceid=278, filesize=12.0 K 2024-12-05T00:43:06,950 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for edbe57733a3fd4ee0e06e7066e75f311 in 1676ms, sequenceid=278, compaction requested=true 2024-12-05T00:43:06,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/15e073c94d9d423d865c5ee007ff30ca, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/c9aaf0d9270945ef911c6d457c6af984, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/bdd1b72a1a034f538c9bf6fa7900eead, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7] to archive 2024-12-05T00:43:06,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:43:06,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/6152a928435c4f6498cd592481c4c1d7 2024-12-05T00:43:06,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/4268534fa050461bb6313fb31e58652f 2024-12-05T00:43:06,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a96288e44e104b6290f2a63d56a44f29 2024-12-05T00:43:06,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/15e073c94d9d423d865c5ee007ff30ca to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/15e073c94d9d423d865c5ee007ff30ca 2024-12-05T00:43:06,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/072d015a367241858309c31f2560790c 2024-12-05T00:43:06,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/757f51bd7b064848bfa1f4b00c16ce94 2024-12-05T00:43:06,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/c9aaf0d9270945ef911c6d457c6af984 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/c9aaf0d9270945ef911c6d457c6af984 2024-12-05T00:43:06,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/1b19ab98d01b489e982f594907f9c2ed 2024-12-05T00:43:06,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/bdd1b72a1a034f538c9bf6fa7900eead to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/bdd1b72a1a034f538c9bf6fa7900eead 2024-12-05T00:43:06,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/f925420bf3a743ea8bf9fc1958c2e918 2024-12-05T00:43:06,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/851ddb19f062447bb61ffcd4a86a98b1 2024-12-05T00:43:06,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/a466aa81ee4e4e97a84779c9fa0af8f7 2024-12-05T00:43:06,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3c8865a5c9ef47ffa0e9dae300ea4155, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb0b5d2f733c4442a97f9cba481be9a7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d67e9ced5aed4e5894f7efe1cd9e7a22, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb99f9a0d9574c47ab86060b00abb3f0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3bd76526ab49446d865d88872fb7018f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/8090a038f059499d931087312a56bd13, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d28c77f1ded84988ac6ff64217580dc2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/49efa8fc9a394887addc056eb79a4620, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/bd3b11dd48e346199e81a7b62448b5d4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d808428d64064f6b98c1e75ca2b79543, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/2456892d1c6c4ffaac0a3ce97868d653, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/85b8c2ec0ac444a5bcc06759e5bb8387] to archive 2024-12-05T00:43:06,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:43:06,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3c8865a5c9ef47ffa0e9dae300ea4155 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3c8865a5c9ef47ffa0e9dae300ea4155 2024-12-05T00:43:06,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb0b5d2f733c4442a97f9cba481be9a7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb0b5d2f733c4442a97f9cba481be9a7 2024-12-05T00:43:06,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d67e9ced5aed4e5894f7efe1cd9e7a22 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d67e9ced5aed4e5894f7efe1cd9e7a22 2024-12-05T00:43:06,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb99f9a0d9574c47ab86060b00abb3f0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/fb99f9a0d9574c47ab86060b00abb3f0 2024-12-05T00:43:06,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3bd76526ab49446d865d88872fb7018f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/3bd76526ab49446d865d88872fb7018f 2024-12-05T00:43:06,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/8090a038f059499d931087312a56bd13 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/8090a038f059499d931087312a56bd13 2024-12-05T00:43:06,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d28c77f1ded84988ac6ff64217580dc2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d28c77f1ded84988ac6ff64217580dc2 2024-12-05T00:43:06,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/49efa8fc9a394887addc056eb79a4620 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/49efa8fc9a394887addc056eb79a4620 2024-12-05T00:43:06,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/bd3b11dd48e346199e81a7b62448b5d4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/bd3b11dd48e346199e81a7b62448b5d4 2024-12-05T00:43:06,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d808428d64064f6b98c1e75ca2b79543 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/d808428d64064f6b98c1e75ca2b79543 2024-12-05T00:43:06,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/2456892d1c6c4ffaac0a3ce97868d653 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/2456892d1c6c4ffaac0a3ce97868d653 2024-12-05T00:43:06,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/85b8c2ec0ac444a5bcc06759e5bb8387 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/85b8c2ec0ac444a5bcc06759e5bb8387 2024-12-05T00:43:06,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/964c729296c44e16af9c12d0e6115aa1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/565772b12cd344f682a8039a73d14f7b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/cd73e683b86e471e9138afe638d469b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/bd482196b5b54c9da0e5a7d0bbd81599, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/61cb08aaebc849349c7b0c2ae9c6baa9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/2000dd124e1841f7862ecce2618ada24, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0801bf9dfaef45b3a13715d91e7d0b64, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/458cf94a50d345f1aa62e728f4a97662, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/583c789d857049d99cea35e05ce22c30, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d2b9c26e68cc45f6ae7a64d4caf9f58f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/710f85766f2f4da58a57b60c43fd1feb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6b3ad713aee24b11ac0cde7d9be0c4cb] to archive 2024-12-05T00:43:06,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:43:06,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/964c729296c44e16af9c12d0e6115aa1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/964c729296c44e16af9c12d0e6115aa1 2024-12-05T00:43:06,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/565772b12cd344f682a8039a73d14f7b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/565772b12cd344f682a8039a73d14f7b 2024-12-05T00:43:06,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/cd73e683b86e471e9138afe638d469b8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/cd73e683b86e471e9138afe638d469b8 2024-12-05T00:43:06,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/bd482196b5b54c9da0e5a7d0bbd81599 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/bd482196b5b54c9da0e5a7d0bbd81599 2024-12-05T00:43:06,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/61cb08aaebc849349c7b0c2ae9c6baa9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/61cb08aaebc849349c7b0c2ae9c6baa9 2024-12-05T00:43:06,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/2000dd124e1841f7862ecce2618ada24 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/2000dd124e1841f7862ecce2618ada24 2024-12-05T00:43:06,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0801bf9dfaef45b3a13715d91e7d0b64 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0801bf9dfaef45b3a13715d91e7d0b64 2024-12-05T00:43:06,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/458cf94a50d345f1aa62e728f4a97662 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/458cf94a50d345f1aa62e728f4a97662 2024-12-05T00:43:06,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/583c789d857049d99cea35e05ce22c30 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/583c789d857049d99cea35e05ce22c30 2024-12-05T00:43:06,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d2b9c26e68cc45f6ae7a64d4caf9f58f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d2b9c26e68cc45f6ae7a64d4caf9f58f 2024-12-05T00:43:06,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/710f85766f2f4da58a57b60c43fd1feb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/710f85766f2f4da58a57b60c43fd1feb 2024-12-05T00:43:06,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6b3ad713aee24b11ac0cde7d9be0c4cb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6b3ad713aee24b11ac0cde7d9be0c4cb 2024-12-05T00:43:06,985 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/recovered.edits/281.seqid, newMaxSeqId=281, maxSeqId=4 2024-12-05T00:43:06,985 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. 2024-12-05T00:43:06,986 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for edbe57733a3fd4ee0e06e7066e75f311: Waiting for close lock at 1733359384275Running coprocessor pre-close hooks at 1733359384275Disabling compacts and flushes for region at 1733359384275Disabling writes for close at 1733359385274 (+999 ms)Obtaining lock to block concurrent updates at 1733359385274Preparing flush snapshotting stores in edbe57733a3fd4ee0e06e7066e75f311 at 1733359385274Finished memstore snapshotting TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311., syncing WAL and waiting on mvcc, flushsize=dataSize=68700, getHeapSize=204720, getOffHeapSize=0, getCellsCount=1500 at 1733359385274Flushing stores of TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311. at 1733359385275 (+1 ms)Flushing edbe57733a3fd4ee0e06e7066e75f311/A: creating writer at 1733359385275Flushing mob file edbe57733a3fd4ee0e06e7066e75f311/A: appending metadata at 1733359385280 (+5 ms)Flushing mob file edbe57733a3fd4ee0e06e7066e75f311/A: closing flushed file at 1733359385280Flushing edbe57733a3fd4ee0e06e7066e75f311/A: appending metadata at 1733359385691 (+411 ms)Flushing edbe57733a3fd4ee0e06e7066e75f311/A: closing flushed file at 1733359385691Flushing edbe57733a3fd4ee0e06e7066e75f311/B: creating writer at 1733359386103 (+412 ms)Flushing edbe57733a3fd4ee0e06e7066e75f311/B: appending metadata at 1733359386106 (+3 ms)Flushing edbe57733a3fd4ee0e06e7066e75f311/B: closing flushed file at 1733359386106Flushing edbe57733a3fd4ee0e06e7066e75f311/C: creating writer at 1733359386520 (+414 ms)Flushing edbe57733a3fd4ee0e06e7066e75f311/C: appending metadata at 1733359386523 (+3 ms)Flushing edbe57733a3fd4ee0e06e7066e75f311/C: closing flushed file at 1733359386523Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a69830: reopening flushed file at 1733359386934 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44a467fd: reopening flushed file at 1733359386940 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a8dafe3: reopening flushed file at 1733359386946 (+6 ms)Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for edbe57733a3fd4ee0e06e7066e75f311 in 1676ms, sequenceid=278, compaction requested=true at 1733359386950 (+4 ms)Writing region close event to WAL at 1733359386983 (+33 ms)Running coprocessor post-close hooks at 1733359386985 (+2 ms)Closed at 1733359386985 2024-12-05T00:43:06,987 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:06,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=edbe57733a3fd4ee0e06e7066e75f311, regionState=CLOSED 2024-12-05T00:43:06,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:43:06,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-05T00:43:06,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure edbe57733a3fd4ee0e06e7066e75f311, server=fea72ea5c4b6,42683,1733359267474 in 2.8680 sec 2024-12-05T00:43:06,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-05T00:43:06,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edbe57733a3fd4ee0e06e7066e75f311, UNASSIGN in 2.8710 sec 2024-12-05T00:43:06,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-05T00:43:06,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8750 sec 2024-12-05T00:43:06,994 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359386994"}]},"ts":"1733359386994"} 2024-12-05T00:43:06,995 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T00:43:06,995 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T00:43:06,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8850 sec 2024-12-05T00:43:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T00:43:08,283 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:08,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T00:43:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,288 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T00:43:08,290 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,292 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,296 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/recovered.edits] 2024-12-05T00:43:08,300 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/0a7d3cb4e2c14c3da56f2754b5908447 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/0a7d3cb4e2c14c3da56f2754b5908447 2024-12-05T00:43:08,302 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/18c6d34939804d338c7fd6848ae486e6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/18c6d34939804d338c7fd6848ae486e6 2024-12-05T00:43:08,304 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/3644772e454047e1b4a1da5a091a2d0b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/3644772e454047e1b4a1da5a091a2d0b 2024-12-05T00:43:08,306 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/8e1ed171408943a39a65dfd8df091398 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/8e1ed171408943a39a65dfd8df091398 2024-12-05T00:43:08,309 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/d515c5a2aa0849d3a51fd6806285a66d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/d515c5a2aa0849d3a51fd6806285a66d 2024-12-05T00:43:08,311 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/e043e31b67ca4831b7aacc727c6ee7ea to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/A/e043e31b67ca4831b7aacc727c6ee7ea 2024-12-05T00:43:08,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/28c10e76027c4c73a72317c5ce393359 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/28c10e76027c4c73a72317c5ce393359 2024-12-05T00:43:08,318 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/34d40dcccf1d4af7987126041a267de8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/34d40dcccf1d4af7987126041a267de8 2024-12-05T00:43:08,320 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/97e379fa3e1f458c9c0551016f50b153 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/97e379fa3e1f458c9c0551016f50b153 2024-12-05T00:43:08,322 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b001ddb6b77e43bf8c0311de09772278 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b001ddb6b77e43bf8c0311de09772278 2024-12-05T00:43:08,324 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b5ca6297d0564b4894f3ed742e25047e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/b5ca6297d0564b4894f3ed742e25047e 2024-12-05T00:43:08,326 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/ef4aaaf7d4c24f42b965226f38b19a9d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/B/ef4aaaf7d4c24f42b965226f38b19a9d 2024-12-05T00:43:08,329 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0a9fca99ba96467282a28d010408d761 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/0a9fca99ba96467282a28d010408d761 2024-12-05T00:43:08,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/59ae71624e034511991931e9a5e79635 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/59ae71624e034511991931e9a5e79635 2024-12-05T00:43:08,333 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6e77167f68bd41ee863aafa7a95854fe to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/6e77167f68bd41ee863aafa7a95854fe 2024-12-05T00:43:08,334 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/847a7d558a9948a098a6765c30d9659d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/847a7d558a9948a098a6765c30d9659d 2024-12-05T00:43:08,336 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d97bbc23f0394d13b24fb0fcbac8d356 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/d97bbc23f0394d13b24fb0fcbac8d356 2024-12-05T00:43:08,338 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/ff33a30947b741dfa240668612661bd7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/C/ff33a30947b741dfa240668612661bd7 2024-12-05T00:43:08,341 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/recovered.edits/281.seqid to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311/recovered.edits/281.seqid 2024-12-05T00:43:08,342 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,342 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T00:43:08,343 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T00:43:08,344 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T00:43:08,347 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205034809dbb8414130b1b9d5fef219f98c_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205034809dbb8414130b1b9d5fef219f98c_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,348 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c76ba14d3bb48c3b58daf500321753d_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c76ba14d3bb48c3b58daf500321753d_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051e7d7f1fde7041cda1709fba4e255bc4_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051e7d7f1fde7041cda1709fba4e255bc4_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205317b9425d637487b9b25b6f9898a7517_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205317b9425d637487b9b25b6f9898a7517_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054c82a4097da048f28776eabb124046ee_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054c82a4097da048f28776eabb124046ee_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120573f9d5b845be44f19e41d8a2df024e5e_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120573f9d5b845be44f19e41d8a2df024e5e_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,353 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205934bbadede5b498db33205fd0eb098e5_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205934bbadede5b498db33205fd0eb098e5_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,353 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a0104f84e1924834aca70e9cdc6cfa35_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a0104f84e1924834aca70e9cdc6cfa35_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,354 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b25e55cb617f4617bfa43af6c50b8dee_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b25e55cb617f4617bfa43af6c50b8dee_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bc3de570a9564e4e9095f82197c02f92_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bc3de570a9564e4e9095f82197c02f92_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205df382c531d994f2c9e13906da89f172b_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205df382c531d994f2c9e13906da89f172b_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,357 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f46b8614082145249aa14724f09373e2_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f46b8614082145249aa14724f09373e2_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fbd6ee8307a0445e8814290df842fd40_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fbd6ee8307a0445e8814290df842fd40_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,359 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fc6b38a1cadc4751a89ca865f81b0b88_edbe57733a3fd4ee0e06e7066e75f311 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fc6b38a1cadc4751a89ca865f81b0b88_edbe57733a3fd4ee0e06e7066e75f311 2024-12-05T00:43:08,359 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T00:43:08,361 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,363 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T00:43:08,364 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T00:43:08,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T00:43:08,365 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733359388365"}]},"ts":"9223372036854775807"} 2024-12-05T00:43:08,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T00:43:08,367 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => edbe57733a3fd4ee0e06e7066e75f311, NAME => 'TestAcidGuarantees,,1733359360393.edbe57733a3fd4ee0e06e7066e75f311.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T00:43:08,367 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T00:43:08,367 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733359388367"}]},"ts":"9223372036854775807"} 2024-12-05T00:43:08,369 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T00:43:08,369 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-12-05T00:43:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T00:43:08,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T00:43:08,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:08,399 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1522171622_22 at /127.0.0.1:60250 [Waiting for operation #730] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-866614165_22 at /127.0.0.1:59388 [Waiting for operation #720] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-866614165_22 at /127.0.0.1:59376 [Waiting for operation #724] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1522171622_22 at /127.0.0.1:35892 [Waiting for operation #1124] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 424) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9335 (was 9401) 2024-12-05T00:43:08,407 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=9334 2024-12-05T00:43:08,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:43:08,409 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:43:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:08,410 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:43:08,410 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:08,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-05T00:43:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-05T00:43:08,411 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:43:08,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742241_1417 (size=1061) 2024-12-05T00:43:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-05T00:43:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-05T00:43:08,822 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:43:08,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742242_1418 (size=53) 2024-12-05T00:43:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-05T00:43:09,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:43:09,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1e9fd1cbd3f1490919d61090b95cb9a3, disabling compactions & flushes 2024-12-05T00:43:09,233 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. after waiting 0 ms 2024-12-05T00:43:09,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,233 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,234 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e9fd1cbd3f1490919d61090b95cb9a3: Waiting for close lock at 1733359389233Disabling compacts and flushes for region at 1733359389233Disabling writes for close at 1733359389233Writing region close event to WAL at 1733359389233Closed at 1733359389233 2024-12-05T00:43:09,236 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:43:09,237 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733359389236"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359389236"}]},"ts":"1733359389236"} 2024-12-05T00:43:09,240 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:43:09,242 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:43:09,242 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359389242"}]},"ts":"1733359389242"} 2024-12-05T00:43:09,244 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T00:43:09,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, ASSIGN}] 2024-12-05T00:43:09,245 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, ASSIGN 2024-12-05T00:43:09,246 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=false 2024-12-05T00:43:09,397 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=1e9fd1cbd3f1490919d61090b95cb9a3, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:09,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, ASSIGN because future has completed 2024-12-05T00:43:09,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:43:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-05T00:43:09,563 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,564 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:43:09,565 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,565 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:43:09,565 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,565 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,568 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,569 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:09,569 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e9fd1cbd3f1490919d61090b95cb9a3 columnFamilyName A 2024-12-05T00:43:09,569 DEBUG [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:09,569 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.HStore(327): Store=1e9fd1cbd3f1490919d61090b95cb9a3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:09,570 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,570 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:09,570 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e9fd1cbd3f1490919d61090b95cb9a3 columnFamilyName B 2024-12-05T00:43:09,571 DEBUG [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:09,571 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.HStore(327): Store=1e9fd1cbd3f1490919d61090b95cb9a3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:09,571 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,572 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:09,572 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e9fd1cbd3f1490919d61090b95cb9a3 columnFamilyName C 2024-12-05T00:43:09,572 DEBUG [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:09,572 INFO [StoreOpener-1e9fd1cbd3f1490919d61090b95cb9a3-1 {}] regionserver.HStore(327): Store=1e9fd1cbd3f1490919d61090b95cb9a3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:09,572 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,572 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,573 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,573 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,574 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,574 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,574 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:43:09,575 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,576 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:43:09,576 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 1e9fd1cbd3f1490919d61090b95cb9a3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69570982, jitterRate=0.036688417196273804}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:43:09,576 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:09,577 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 1e9fd1cbd3f1490919d61090b95cb9a3: Running coprocessor pre-open hook at 1733359389565Writing region info on filesystem at 1733359389565Initializing all the Stores at 1733359389567 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359389567Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359389567Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359389567Cleaning up temporary data from old regions at 1733359389574 (+7 ms)Running coprocessor post-open hooks at 1733359389576 (+2 ms)Region opened successfully at 1733359389577 (+1 ms) 2024-12-05T00:43:09,577 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., pid=120, masterSystemTime=1733359389555 2024-12-05T00:43:09,579 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,579 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:09,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=1e9fd1cbd3f1490919d61090b95cb9a3, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:09,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:43:09,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T00:43:09,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 in 181 msec 2024-12-05T00:43:09,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-05T00:43:09,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, ASSIGN in 339 msec 2024-12-05T00:43:09,585 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:43:09,585 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359389585"}]},"ts":"1733359389585"} 2024-12-05T00:43:09,586 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T00:43:09,587 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:43:09,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-12-05T00:43:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-05T00:43:10,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:10,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16af2234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5105d4e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,565 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f67ecb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,568 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:43:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:43:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45bb3b03 2024-12-05T00:43:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:43:10,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:43:10,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695dbdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,571 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705bae02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225a8601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594674bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c69a687, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a961e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,576 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af227b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ac4b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bcdf9fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ed9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1f545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d7d20d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5585b4cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ae0787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e15d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e7057d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1899d6fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d2c903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e42da9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c71cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117399b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aba3326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5955894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b33a176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:10,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:10,595 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc441a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:10,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c710ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:10,597 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,598 DEBUG [Thread-1896 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,598 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,598 DEBUG [Thread-1890 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,598 DEBUG [Thread-1894 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,598 DEBUG [Thread-1892 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,599 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,600 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,601 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,601 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:10,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:10,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2] 2024-12-05T00:43:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-05T00:43:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:10,610 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:10,610 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:10,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:10,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:10,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/69c1f6810121462ea77d2e462fc5afe3 is 50, key is test_row_0/A:col10/1733359390609/Put/seqid=0 2024-12-05T00:43:10,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742243_1419 (size=14341) 2024-12-05T00:43:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733359390628, max timestamp=1733359390643Num uniques 859; 2024-12-05T00:43:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733359390628, max timestamp=1733359390643Num uniques 859; 2024-12-05T00:43:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T00:43:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T00:43:10,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733359390628, max timestamp=1733359390643Num uniques 859; 2024-12-05T00:43:10,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T00:43:10,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359450659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359450665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359450667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359450668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359450669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:10,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:10,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:10,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359450772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359450781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359450781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359450781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359450781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:10,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:10,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359450981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359450991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359450991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359450991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359450992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/69c1f6810121462ea77d2e462fc5afe3 2024-12-05T00:43:11,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/9240ddb30a5b44ae8831bc9b44595e16 is 50, key is test_row_0/B:col10/1733359390609/Put/seqid=0 2024-12-05T00:43:11,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742244_1420 (size=12001) 2024-12-05T00:43:11,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:11,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:11,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359451292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359451301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359451301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359451301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359451301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/9240ddb30a5b44ae8831bc9b44595e16 2024-12-05T00:43:11,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/7b883a999fce46a99f886cfed82372c1 is 50, key is test_row_0/C:col10/1733359390609/Put/seqid=0 2024-12-05T00:43:11,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742245_1421 (size=12001) 2024-12-05T00:43:11,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359451811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359451811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359451811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359451822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359451822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:11,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:11,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:11,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/7b883a999fce46a99f886cfed82372c1 2024-12-05T00:43:11,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/69c1f6810121462ea77d2e462fc5afe3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/69c1f6810121462ea77d2e462fc5afe3 2024-12-05T00:43:11,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/69c1f6810121462ea77d2e462fc5afe3, entries=200, sequenceid=11, filesize=14.0 K 2024-12-05T00:43:11,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/9240ddb30a5b44ae8831bc9b44595e16 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9240ddb30a5b44ae8831bc9b44595e16 2024-12-05T00:43:11,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9240ddb30a5b44ae8831bc9b44595e16, entries=150, sequenceid=11, filesize=11.7 K 2024-12-05T00:43:11,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/7b883a999fce46a99f886cfed82372c1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/7b883a999fce46a99f886cfed82372c1 2024-12-05T00:43:11,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/7b883a999fce46a99f886cfed82372c1, entries=150, sequenceid=11, filesize=11.7 K 2024-12-05T00:43:11,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1289ms, sequenceid=11, compaction requested=false 2024-12-05T00:43:11,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:11,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:11,984 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:11,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:11,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/656c4aaacde24994939b24db7a22d9a4 is 50, key is test_row_0/A:col10/1733359390668/Put/seqid=0 2024-12-05T00:43:11,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742246_1422 (size=12001) 2024-12-05T00:43:11,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/656c4aaacde24994939b24db7a22d9a4 2024-12-05T00:43:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/b40711c52400493dbf832cae95177321 is 50, key is test_row_0/B:col10/1733359390668/Put/seqid=0 2024-12-05T00:43:12,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742247_1423 (size=12001) 2024-12-05T00:43:12,406 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/b40711c52400493dbf832cae95177321 2024-12-05T00:43:12,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/508dac27af644762b588ee8453c660ab is 50, key is test_row_0/C:col10/1733359390668/Put/seqid=0 2024-12-05T00:43:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742248_1424 (size=12001) 2024-12-05T00:43:12,505 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:43:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:12,816 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/508dac27af644762b588ee8453c660ab 2024-12-05T00:43:12,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/656c4aaacde24994939b24db7a22d9a4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/656c4aaacde24994939b24db7a22d9a4 2024-12-05T00:43:12,821 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/656c4aaacde24994939b24db7a22d9a4, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T00:43:12,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/b40711c52400493dbf832cae95177321 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/b40711c52400493dbf832cae95177321 2024-12-05T00:43:12,825 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/b40711c52400493dbf832cae95177321, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T00:43:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/508dac27af644762b588ee8453c660ab as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/508dac27af644762b588ee8453c660ab 2024-12-05T00:43:12,829 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/508dac27af644762b588ee8453c660ab, entries=150, sequenceid=46, filesize=11.7 K 2024-12-05T00:43:12,849 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=0 B/0 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 865ms, sequenceid=46, compaction requested=false 2024-12-05T00:43:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:12,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T00:43:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-05T00:43:12,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T00:43:12,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-12-05T00:43:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:12,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:12,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2520 sec 2024-12-05T00:43:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:12,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4064bc1131554e41b142d64aa620f6f3 is 50, key is test_row_0/A:col10/1733359392856/Put/seqid=0 2024-12-05T00:43:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742249_1425 (size=14341) 2024-12-05T00:43:12,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4064bc1131554e41b142d64aa620f6f3 2024-12-05T00:43:12,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/afd4f758845f4fbdafe35364706e161b is 50, key is test_row_0/B:col10/1733359392856/Put/seqid=0 2024-12-05T00:43:12,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742250_1426 (size=12001) 2024-12-05T00:43:12,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359392871, max timestamp=1733359392894Num uniques 859; 2024-12-05T00:43:12,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359392871, max timestamp=1733359392894Num uniques 859; 2024-12-05T00:43:12,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359392871, max timestamp=1733359392894Num uniques 859; 2024-12-05T00:43:12,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:43:12,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:43:12,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:43:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359452914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359452919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359452926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:12,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:12,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359452928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359452928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359453032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359453032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359453032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359453051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359453051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359453242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359453242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359453242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359453261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359453262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/afd4f758845f4fbdafe35364706e161b 2024-12-05T00:43:13,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/4a2a80ee79934f4098c0b7e80c72ec4c is 50, key is test_row_0/C:col10/1733359392856/Put/seqid=0 2024-12-05T00:43:13,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742251_1427 (size=12001) 2024-12-05T00:43:13,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359453552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359453552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359453552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359453571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359453571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:13,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/4a2a80ee79934f4098c0b7e80c72ec4c 2024-12-05T00:43:13,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4064bc1131554e41b142d64aa620f6f3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4064bc1131554e41b142d64aa620f6f3 2024-12-05T00:43:13,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4064bc1131554e41b142d64aa620f6f3, entries=200, sequenceid=57, filesize=14.0 K 2024-12-05T00:43:13,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/afd4f758845f4fbdafe35364706e161b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/afd4f758845f4fbdafe35364706e161b 2024-12-05T00:43:13,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/afd4f758845f4fbdafe35364706e161b, entries=150, sequenceid=57, filesize=11.7 K 2024-12-05T00:43:13,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/4a2a80ee79934f4098c0b7e80c72ec4c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/4a2a80ee79934f4098c0b7e80c72ec4c 2024-12-05T00:43:13,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/4a2a80ee79934f4098c0b7e80c72ec4c, entries=150, sequenceid=57, filesize=11.7 K 2024-12-05T00:43:13,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 858ms, sequenceid=57, compaction requested=true 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:13,714 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:13,714 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:13,715 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:13,715 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:13,715 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/B is initiating minor compaction (all files) 2024-12-05T00:43:13,715 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/A is initiating minor compaction (all files) 2024-12-05T00:43:13,715 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/A in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:13,715 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/B in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:13,715 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9240ddb30a5b44ae8831bc9b44595e16, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/b40711c52400493dbf832cae95177321, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/afd4f758845f4fbdafe35364706e161b] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=35.2 K 2024-12-05T00:43:13,715 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/69c1f6810121462ea77d2e462fc5afe3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/656c4aaacde24994939b24db7a22d9a4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4064bc1131554e41b142d64aa620f6f3] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=39.7 K 2024-12-05T00:43:13,716 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9240ddb30a5b44ae8831bc9b44595e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733359390607 2024-12-05T00:43:13,716 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c1f6810121462ea77d2e462fc5afe3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733359390607 2024-12-05T00:43:13,716 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656c4aaacde24994939b24db7a22d9a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359390658 2024-12-05T00:43:13,716 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b40711c52400493dbf832cae95177321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359390658 2024-12-05T00:43:13,716 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4064bc1131554e41b142d64aa620f6f3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733359392824 2024-12-05T00:43:13,716 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting afd4f758845f4fbdafe35364706e161b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733359392824 2024-12-05T00:43:13,721 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#A#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:13,721 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:13,721 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/3455dbffa2884b82802d3e4cbe4cc162 is 50, key is test_row_0/B:col10/1733359392856/Put/seqid=0 2024-12-05T00:43:13,721 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/9e3e1839c84a47d98911049f6675fe02 is 50, key is test_row_0/A:col10/1733359392856/Put/seqid=0 2024-12-05T00:43:13,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742253_1429 (size=12104) 2024-12-05T00:43:13,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742252_1428 (size=12104) 2024-12-05T00:43:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:14,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-05T00:43:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:14,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/1f6854cb0d8d40728534ed6799a4dc1c is 50, key is test_row_0/A:col10/1733359394064/Put/seqid=0 2024-12-05T00:43:14,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742254_1430 (size=14341) 2024-12-05T00:43:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359454080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359454080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359454081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359454082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359454092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,134 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/9e3e1839c84a47d98911049f6675fe02 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/9e3e1839c84a47d98911049f6675fe02 2024-12-05T00:43:14,137 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/3455dbffa2884b82802d3e4cbe4cc162 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/3455dbffa2884b82802d3e4cbe4cc162 2024-12-05T00:43:14,137 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/A of 1e9fd1cbd3f1490919d61090b95cb9a3 into 9e3e1839c84a47d98911049f6675fe02(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:14,137 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:14,137 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/A, priority=13, startTime=1733359393714; duration=0sec 2024-12-05T00:43:14,137 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:14,137 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:A 2024-12-05T00:43:14,138 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:14,138 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:14,138 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/C is initiating minor compaction (all files) 2024-12-05T00:43:14,138 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/C in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:14,138 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/7b883a999fce46a99f886cfed82372c1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/508dac27af644762b588ee8453c660ab, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/4a2a80ee79934f4098c0b7e80c72ec4c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=35.2 K 2024-12-05T00:43:14,139 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b883a999fce46a99f886cfed82372c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733359390607 2024-12-05T00:43:14,139 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508dac27af644762b588ee8453c660ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733359390658 2024-12-05T00:43:14,139 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a2a80ee79934f4098c0b7e80c72ec4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733359392824 2024-12-05T00:43:14,141 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/B of 1e9fd1cbd3f1490919d61090b95cb9a3 into 3455dbffa2884b82802d3e4cbe4cc162(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:14,141 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:14,141 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/B, priority=13, startTime=1733359393714; duration=0sec 2024-12-05T00:43:14,141 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:14,141 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:B 2024-12-05T00:43:14,144 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#C#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:14,144 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/af3de24282bb401cbe03ba3c04352800 is 50, key is test_row_0/C:col10/1733359392856/Put/seqid=0 2024-12-05T00:43:14,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742255_1431 (size=12104) 2024-12-05T00:43:14,151 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/af3de24282bb401cbe03ba3c04352800 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/af3de24282bb401cbe03ba3c04352800 2024-12-05T00:43:14,154 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/C of 1e9fd1cbd3f1490919d61090b95cb9a3 into af3de24282bb401cbe03ba3c04352800(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:14,154 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:14,154 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/C, priority=13, startTime=1733359393714; duration=0sec 2024-12-05T00:43:14,154 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:14,154 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:C 2024-12-05T00:43:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359454192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359454192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359454202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359454402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359454402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359454412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/1f6854cb0d8d40728534ed6799a4dc1c 2024-12-05T00:43:14,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/182caa79e13c4fcb9b0371d4de9da238 is 50, key is test_row_0/B:col10/1733359394064/Put/seqid=0 2024-12-05T00:43:14,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742256_1432 (size=12001) 2024-12-05T00:43:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359454711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359454711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359454721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T00:43:14,762 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:14,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-05T00:43:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T00:43:14,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:14,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:14,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T00:43:14,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/182caa79e13c4fcb9b0371d4de9da238 2024-12-05T00:43:14,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/23de92e861fa4580a9c88a0cb0a60eaf is 50, key is test_row_0/C:col10/1733359394064/Put/seqid=0 2024-12-05T00:43:14,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742257_1433 (size=12001) 2024-12-05T00:43:14,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T00:43:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:14,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:14,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T00:43:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:15,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T00:43:15,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359455102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:15,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359455102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:15,107 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:15,107 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:15,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T00:43:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:15,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:15,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359455222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:15,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359455222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:15,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359455231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:15,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:15,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:15,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/23de92e861fa4580a9c88a0cb0a60eaf 2024-12-05T00:43:15,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/1f6854cb0d8d40728534ed6799a4dc1c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/1f6854cb0d8d40728534ed6799a4dc1c 2024-12-05T00:43:15,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/1f6854cb0d8d40728534ed6799a4dc1c, entries=200, sequenceid=93, filesize=14.0 K 2024-12-05T00:43:15,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/182caa79e13c4fcb9b0371d4de9da238 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/182caa79e13c4fcb9b0371d4de9da238 2024-12-05T00:43:15,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/182caa79e13c4fcb9b0371d4de9da238, entries=150, sequenceid=93, filesize=11.7 K 2024-12-05T00:43:15,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/23de92e861fa4580a9c88a0cb0a60eaf as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/23de92e861fa4580a9c88a0cb0a60eaf 2024-12-05T00:43:15,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/23de92e861fa4580a9c88a0cb0a60eaf, entries=150, sequenceid=93, filesize=11.7 K 2024-12-05T00:43:15,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1252ms, sequenceid=93, compaction requested=false 2024-12-05T00:43:15,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:15,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T00:43:15,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:15,375 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T00:43:15,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/a86850cab2934ad8b0348c8118bb0931 is 50, key is test_row_0/A:col10/1733359394078/Put/seqid=0 2024-12-05T00:43:15,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742258_1434 (size=12001) 2024-12-05T00:43:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T00:43:15,784 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/a86850cab2934ad8b0348c8118bb0931 2024-12-05T00:43:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/e7c3cf49fa9b41cbba5178d942226947 is 50, key is test_row_0/B:col10/1733359394078/Put/seqid=0 2024-12-05T00:43:15,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742259_1435 (size=12001) 2024-12-05T00:43:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T00:43:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:16,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:16,236 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/e7c3cf49fa9b41cbba5178d942226947 2024-12-05T00:43:16,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/0151032053b14e25967af591497e4475 is 50, key is test_row_0/C:col10/1733359394078/Put/seqid=0 2024-12-05T00:43:16,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742260_1436 (size=12001) 2024-12-05T00:43:16,245 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/0151032053b14e25967af591497e4475 2024-12-05T00:43:16,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/a86850cab2934ad8b0348c8118bb0931 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/a86850cab2934ad8b0348c8118bb0931 2024-12-05T00:43:16,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/a86850cab2934ad8b0348c8118bb0931, entries=150, sequenceid=105, filesize=11.7 K 2024-12-05T00:43:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/e7c3cf49fa9b41cbba5178d942226947 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/e7c3cf49fa9b41cbba5178d942226947 2024-12-05T00:43:16,254 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/e7c3cf49fa9b41cbba5178d942226947, entries=150, sequenceid=105, filesize=11.7 K 2024-12-05T00:43:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/0151032053b14e25967af591497e4475 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0151032053b14e25967af591497e4475 2024-12-05T00:43:16,258 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0151032053b14e25967af591497e4475, entries=150, sequenceid=105, filesize=11.7 K 2024-12-05T00:43:16,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=40.25 KB/41220 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 884ms, sequenceid=105, compaction requested=true 2024-12-05T00:43:16,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:16,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:16,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T00:43:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-05T00:43:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:16,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:16,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T00:43:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:16,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-12-05T00:43:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:16,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4990 sec 2024-12-05T00:43:16,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/ac906667e4ef4290af27f1822d21f351 is 50, key is test_row_0/A:col10/1733359396246/Put/seqid=0 2024-12-05T00:43:16,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742261_1437 (size=14341) 2024-12-05T00:43:16,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359396286, max timestamp=1733359396312Num uniques 859; 2024-12-05T00:43:16,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359456307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359396286, max timestamp=1733359396312Num uniques 859; 2024-12-05T00:43:16,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359396286, max timestamp=1733359396312Num uniques 859; 2024-12-05T00:43:16,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:16,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359456380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359456381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359456421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359456491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359456491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359456631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/ac906667e4ef4290af27f1822d21f351 2024-12-05T00:43:16,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/8f531bf3ae094073a6a42b09c38891a1 is 50, key is test_row_0/B:col10/1733359396246/Put/seqid=0 2024-12-05T00:43:16,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742262_1438 (size=12001) 2024-12-05T00:43:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359456702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359456702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T00:43:16,912 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:16,913 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-05T00:43:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T00:43:16,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:16,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:16,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:16,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359456951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359457012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359457012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T00:43:17,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T00:43:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/8f531bf3ae094073a6a42b09c38891a1 2024-12-05T00:43:17,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/77210a3787cd421e89917aea5862039f is 50, key is test_row_0/C:col10/1733359396246/Put/seqid=0 2024-12-05T00:43:17,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742263_1439 (size=12001) 2024-12-05T00:43:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359457121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359457121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,128 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,128 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T00:43:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T00:43:17,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T00:43:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359457462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/77210a3787cd421e89917aea5862039f 2024-12-05T00:43:17,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/ac906667e4ef4290af27f1822d21f351 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ac906667e4ef4290af27f1822d21f351 2024-12-05T00:43:17,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ac906667e4ef4290af27f1822d21f351, entries=200, sequenceid=115, filesize=14.0 K 2024-12-05T00:43:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/8f531bf3ae094073a6a42b09c38891a1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/8f531bf3ae094073a6a42b09c38891a1 2024-12-05T00:43:17,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/8f531bf3ae094073a6a42b09c38891a1, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T00:43:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/77210a3787cd421e89917aea5862039f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/77210a3787cd421e89917aea5862039f 2024-12-05T00:43:17,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/77210a3787cd421e89917aea5862039f, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T00:43:17,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1244ms, sequenceid=115, compaction requested=true 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:17,505 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:17,505 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:17,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:17,506 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:17,506 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:17,506 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/B is initiating minor compaction (all files) 2024-12-05T00:43:17,506 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/A is initiating minor compaction (all files) 2024-12-05T00:43:17,506 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/A in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,506 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/B in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,506 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/3455dbffa2884b82802d3e4cbe4cc162, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/182caa79e13c4fcb9b0371d4de9da238, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/e7c3cf49fa9b41cbba5178d942226947, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/8f531bf3ae094073a6a42b09c38891a1] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=47.0 K 2024-12-05T00:43:17,506 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/9e3e1839c84a47d98911049f6675fe02, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/1f6854cb0d8d40728534ed6799a4dc1c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/a86850cab2934ad8b0348c8118bb0931, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ac906667e4ef4290af27f1822d21f351] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=51.5 K 2024-12-05T00:43:17,506 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3455dbffa2884b82802d3e4cbe4cc162, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733359392824 2024-12-05T00:43:17,506 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e3e1839c84a47d98911049f6675fe02, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733359392824 2024-12-05T00:43:17,507 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 182caa79e13c4fcb9b0371d4de9da238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733359392926 2024-12-05T00:43:17,507 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f6854cb0d8d40728534ed6799a4dc1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733359392911 2024-12-05T00:43:17,507 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c3cf49fa9b41cbba5178d942226947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733359394078 2024-12-05T00:43:17,507 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting a86850cab2934ad8b0348c8118bb0931, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733359394078 2024-12-05T00:43:17,507 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f531bf3ae094073a6a42b09c38891a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733359396239 2024-12-05T00:43:17,507 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac906667e4ef4290af27f1822d21f351, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733359396239 2024-12-05T00:43:17,512 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:17,513 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/eeb29d44b72c47029677e3fa95f04467 is 50, key is test_row_0/B:col10/1733359396246/Put/seqid=0 2024-12-05T00:43:17,513 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#A#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:17,514 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/7f3d274b942f49958fa81719c43c3834 is 50, key is test_row_0/A:col10/1733359396246/Put/seqid=0 2024-12-05T00:43:17,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742264_1440 (size=12241) 2024-12-05T00:43:17,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742265_1441 (size=12241) 2024-12-05T00:43:17,519 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/eeb29d44b72c47029677e3fa95f04467 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/eeb29d44b72c47029677e3fa95f04467 2024-12-05T00:43:17,522 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/B of 1e9fd1cbd3f1490919d61090b95cb9a3 into eeb29d44b72c47029677e3fa95f04467(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:17,522 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:17,522 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/B, priority=12, startTime=1733359397505; duration=0sec 2024-12-05T00:43:17,522 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:17,522 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:B 2024-12-05T00:43:17,522 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:17,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:17,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/C is initiating minor compaction (all files) 2024-12-05T00:43:17,523 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/C in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,523 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/af3de24282bb401cbe03ba3c04352800, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/23de92e861fa4580a9c88a0cb0a60eaf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0151032053b14e25967af591497e4475, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/77210a3787cd421e89917aea5862039f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=47.0 K 2024-12-05T00:43:17,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting af3de24282bb401cbe03ba3c04352800, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733359392824 2024-12-05T00:43:17,523 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 23de92e861fa4580a9c88a0cb0a60eaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733359392926 2024-12-05T00:43:17,524 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0151032053b14e25967af591497e4475, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733359394078 2024-12-05T00:43:17,524 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 77210a3787cd421e89917aea5862039f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733359396239 2024-12-05T00:43:17,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T00:43:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:17,524 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T00:43:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:17,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:17,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:17,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:17,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:17,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/54606e2d3d4d46408a97e5a647392cda is 50, key is test_row_0/A:col10/1733359396379/Put/seqid=0 2024-12-05T00:43:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:17,533 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:17,534 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/2c21f3c7db1f4d3ca4c8efc8972edb55 is 50, key is test_row_0/C:col10/1733359396246/Put/seqid=0 2024-12-05T00:43:17,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742266_1442 (size=12151) 2024-12-05T00:43:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T00:43:17,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742267_1443 (size=12241) 2024-12-05T00:43:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359457559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359457563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359457672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359457682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359457881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359457892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:17,920 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/7f3d274b942f49958fa81719c43c3834 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/7f3d274b942f49958fa81719c43c3834 2024-12-05T00:43:17,923 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/A of 1e9fd1cbd3f1490919d61090b95cb9a3 into 7f3d274b942f49958fa81719c43c3834(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:17,923 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:17,923 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/A, priority=12, startTime=1733359397505; duration=0sec 2024-12-05T00:43:17,924 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:17,924 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:A 2024-12-05T00:43:17,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/54606e2d3d4d46408a97e5a647392cda 2024-12-05T00:43:17,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/37fa0e2981c346738e85c1d90711eaf0 is 50, key is test_row_0/B:col10/1733359396379/Put/seqid=0 2024-12-05T00:43:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742268_1444 (size=12151) 2024-12-05T00:43:17,953 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/2c21f3c7db1f4d3ca4c8efc8972edb55 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/2c21f3c7db1f4d3ca4c8efc8972edb55 2024-12-05T00:43:17,957 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/C of 1e9fd1cbd3f1490919d61090b95cb9a3 into 2c21f3c7db1f4d3ca4c8efc8972edb55(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:17,957 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:17,957 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/C, priority=12, startTime=1733359397505; duration=0sec 2024-12-05T00:43:17,957 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:17,957 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:C 2024-12-05T00:43:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T00:43:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359458192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:18,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359458201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:18,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:18,348 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/37fa0e2981c346738e85c1d90711eaf0 2024-12-05T00:43:18,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/b8ff31623ed443eb814c1f1eb9778f2b is 50, key is test_row_0/C:col10/1733359396379/Put/seqid=0 2024-12-05T00:43:18,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742269_1445 (size=12151) 2024-12-05T00:43:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359458481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:18,483 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359458702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:18,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359458711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:18,758 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/b8ff31623ed443eb814c1f1eb9778f2b 2024-12-05T00:43:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/54606e2d3d4d46408a97e5a647392cda as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/54606e2d3d4d46408a97e5a647392cda 2024-12-05T00:43:18,763 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/54606e2d3d4d46408a97e5a647392cda, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T00:43:18,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/37fa0e2981c346738e85c1d90711eaf0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/37fa0e2981c346738e85c1d90711eaf0 2024-12-05T00:43:18,767 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/37fa0e2981c346738e85c1d90711eaf0, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T00:43:18,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/b8ff31623ed443eb814c1f1eb9778f2b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b8ff31623ed443eb814c1f1eb9778f2b 2024-12-05T00:43:18,770 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b8ff31623ed443eb814c1f1eb9778f2b, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T00:43:18,771 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1247ms, sequenceid=152, compaction requested=false 2024-12-05T00:43:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-05T00:43:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-05T00:43:18,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-05T00:43:18,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-12-05T00:43:18,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8610 sec 2024-12-05T00:43:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T00:43:19,062 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:19,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-05T00:43:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T00:43:19,064 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:19,064 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:19,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T00:43:19,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T00:43:19,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:19,216 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:19,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/8d1064c7277045cba3f12615631bd125 is 50, key is test_row_0/A:col10/1733359397561/Put/seqid=0 2024-12-05T00:43:19,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742270_1446 (size=12151) 2024-12-05T00:43:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T00:43:19,625 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/8d1064c7277045cba3f12615631bd125 2024-12-05T00:43:19,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/0aa4769d8f834ad2a5a43fa2ebea1975 is 50, key is test_row_0/B:col10/1733359397561/Put/seqid=0 2024-12-05T00:43:19,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742271_1447 (size=12151) 2024-12-05T00:43:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T00:43:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:19,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:19,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359399760, max timestamp=1733359399791Num uniques 859; 2024-12-05T00:43:19,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:19,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359399760, max timestamp=1733359399791Num uniques 859; 2024-12-05T00:43:19,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359399760, max timestamp=1733359399791Num uniques 859; 2024-12-05T00:43:19,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:19,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:19,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359459838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359459843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359459951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:19,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359459961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,040 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/0aa4769d8f834ad2a5a43fa2ebea1975 2024-12-05T00:43:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/eec84de374ec4174a53dcc3a6765eacb is 50, key is test_row_0/C:col10/1733359397561/Put/seqid=0 2024-12-05T00:43:20,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742272_1448 (size=12151) 2024-12-05T00:43:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359460162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359460181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T00:43:20,449 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/eec84de374ec4174a53dcc3a6765eacb 2024-12-05T00:43:20,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/8d1064c7277045cba3f12615631bd125 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/8d1064c7277045cba3f12615631bd125 2024-12-05T00:43:20,454 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/8d1064c7277045cba3f12615631bd125, entries=150, sequenceid=164, filesize=11.9 K 2024-12-05T00:43:20,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/0aa4769d8f834ad2a5a43fa2ebea1975 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0aa4769d8f834ad2a5a43fa2ebea1975 2024-12-05T00:43:20,457 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0aa4769d8f834ad2a5a43fa2ebea1975, entries=150, sequenceid=164, filesize=11.9 K 2024-12-05T00:43:20,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/eec84de374ec4174a53dcc3a6765eacb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/eec84de374ec4174a53dcc3a6765eacb 2024-12-05T00:43:20,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/eec84de374ec4174a53dcc3a6765eacb, entries=150, sequenceid=164, filesize=11.9 K 2024-12-05T00:43:20,462 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1245ms, sequenceid=164, compaction requested=true 2024-12-05T00:43:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-05T00:43:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-05T00:43:20,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T00:43:20,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-12-05T00:43:20,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4030 sec 2024-12-05T00:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:20,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T00:43:20,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:20,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:20,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:20,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:20,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:20,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:20,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/fc252732482d45fa98dcfda24a08bbf4 is 50, key is test_row_0/A:col10/1733359399841/Put/seqid=0 2024-12-05T00:43:20,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742273_1449 (size=14541) 2024-12-05T00:43:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359460500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359460505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359460505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359460612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359460621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359460621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359460822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359460832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359460832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:20,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/fc252732482d45fa98dcfda24a08bbf4 2024-12-05T00:43:20,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/6023af602467480eb45451c83a1961e0 is 50, key is test_row_0/B:col10/1733359399841/Put/seqid=0 2024-12-05T00:43:20,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742274_1450 (size=12151) 2024-12-05T00:43:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1733359461141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359461141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359461141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,145 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359461152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1733359461171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,177 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T00:43:21,222 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:21,222 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-05T00:43:21,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T00:43:21,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:21,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:21,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/6023af602467480eb45451c83a1961e0 2024-12-05T00:43:21,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/6c8253d58ef04fff94af23eccc77954a is 50, key is test_row_0/C:col10/1733359399841/Put/seqid=0 2024-12-05T00:43:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742275_1451 (size=12151) 2024-12-05T00:43:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T00:43:21,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T00:43:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T00:43:21,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:21,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T00:43:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359461652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359461652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359461661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:21,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T00:43:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:21,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/6c8253d58ef04fff94af23eccc77954a 2024-12-05T00:43:21,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/fc252732482d45fa98dcfda24a08bbf4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/fc252732482d45fa98dcfda24a08bbf4 2024-12-05T00:43:21,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/fc252732482d45fa98dcfda24a08bbf4, entries=200, sequenceid=201, filesize=14.2 K 2024-12-05T00:43:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/6023af602467480eb45451c83a1961e0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/6023af602467480eb45451c83a1961e0 2024-12-05T00:43:21,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/6023af602467480eb45451c83a1961e0, entries=150, sequenceid=201, filesize=11.9 K 2024-12-05T00:43:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/6c8253d58ef04fff94af23eccc77954a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/6c8253d58ef04fff94af23eccc77954a 2024-12-05T00:43:21,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/6c8253d58ef04fff94af23eccc77954a, entries=150, sequenceid=201, filesize=11.9 K 2024-12-05T00:43:21,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1239ms, sequenceid=201, compaction requested=true 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:21,724 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:21,724 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:21,726 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:21,726 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:21,726 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/B is initiating minor compaction (all files) 2024-12-05T00:43:21,726 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/A is initiating minor compaction (all files) 2024-12-05T00:43:21,726 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/B in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,726 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/A in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,726 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/7f3d274b942f49958fa81719c43c3834, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/54606e2d3d4d46408a97e5a647392cda, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/8d1064c7277045cba3f12615631bd125, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/fc252732482d45fa98dcfda24a08bbf4] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=49.9 K 2024-12-05T00:43:21,726 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/eeb29d44b72c47029677e3fa95f04467, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/37fa0e2981c346738e85c1d90711eaf0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0aa4769d8f834ad2a5a43fa2ebea1975, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/6023af602467480eb45451c83a1961e0] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=47.6 K 2024-12-05T00:43:21,726 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb29d44b72c47029677e3fa95f04467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733359396239 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3d274b942f49958fa81719c43c3834, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733359396239 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 37fa0e2981c346738e85c1d90711eaf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733359396353 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54606e2d3d4d46408a97e5a647392cda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733359396353 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa4769d8f834ad2a5a43fa2ebea1975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733359397541 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d1064c7277045cba3f12615631bd125, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733359397541 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6023af602467480eb45451c83a1961e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733359399830 2024-12-05T00:43:21,727 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc252732482d45fa98dcfda24a08bbf4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733359399830 2024-12-05T00:43:21,733 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:21,734 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/24917594092946dfa915c3bc50d0077c is 50, key is test_row_0/A:col10/1733359399841/Put/seqid=0 2024-12-05T00:43:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742276_1452 (size=12527) 2024-12-05T00:43:21,738 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:21,738 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/0bed1e53ad254a4abb11223c88969f46 is 50, key is test_row_0/B:col10/1733359399841/Put/seqid=0 2024-12-05T00:43:21,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742277_1453 (size=12527) 2024-12-05T00:43:21,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T00:43:21,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:21,834 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T00:43:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5e1a8db3966b4ee8b52e096b21b2d661 is 50, key is test_row_0/A:col10/1733359400504/Put/seqid=0 2024-12-05T00:43:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742278_1454 (size=12151) 2024-12-05T00:43:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T00:43:22,142 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/24917594092946dfa915c3bc50d0077c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/24917594092946dfa915c3bc50d0077c 2024-12-05T00:43:22,147 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/0bed1e53ad254a4abb11223c88969f46 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0bed1e53ad254a4abb11223c88969f46 2024-12-05T00:43:22,154 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/A of 1e9fd1cbd3f1490919d61090b95cb9a3 into 24917594092946dfa915c3bc50d0077c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:22,154 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/B of 1e9fd1cbd3f1490919d61090b95cb9a3 into 0bed1e53ad254a4abb11223c88969f46(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:22,154 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/B, priority=12, startTime=1733359401724; duration=0sec 2024-12-05T00:43:22,154 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/A, priority=12, startTime=1733359401724; duration=0sec 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:B 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:A 2024-12-05T00:43:22,154 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:22,155 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:22,155 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/C is initiating minor compaction (all files) 2024-12-05T00:43:22,155 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/C in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:22,155 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/2c21f3c7db1f4d3ca4c8efc8972edb55, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b8ff31623ed443eb814c1f1eb9778f2b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/eec84de374ec4174a53dcc3a6765eacb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/6c8253d58ef04fff94af23eccc77954a] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=47.6 K 2024-12-05T00:43:22,156 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c21f3c7db1f4d3ca4c8efc8972edb55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733359396239 2024-12-05T00:43:22,156 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ff31623ed443eb814c1f1eb9778f2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733359396353 2024-12-05T00:43:22,157 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting eec84de374ec4174a53dcc3a6765eacb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733359397541 2024-12-05T00:43:22,157 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c8253d58ef04fff94af23eccc77954a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733359399830 2024-12-05T00:43:22,163 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:22,164 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/9c00137b1bbb403fb04c6004095a863f is 50, key is test_row_0/C:col10/1733359399841/Put/seqid=0 2024-12-05T00:43:22,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742279_1455 (size=12527) 2024-12-05T00:43:22,241 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5e1a8db3966b4ee8b52e096b21b2d661 2024-12-05T00:43:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/01b8afb3c7de4ba195c685894354c378 is 50, key is test_row_0/B:col10/1733359400504/Put/seqid=0 2024-12-05T00:43:22,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742280_1456 (size=12151) 2024-12-05T00:43:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T00:43:22,574 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/9c00137b1bbb403fb04c6004095a863f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/9c00137b1bbb403fb04c6004095a863f 2024-12-05T00:43:22,579 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/C of 1e9fd1cbd3f1490919d61090b95cb9a3 into 9c00137b1bbb403fb04c6004095a863f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:22,579 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:22,579 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/C, priority=12, startTime=1733359401724; duration=0sec 2024-12-05T00:43:22,579 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:22,579 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:C 2024-12-05T00:43:22,653 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/01b8afb3c7de4ba195c685894354c378 2024-12-05T00:43:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/ee3302bcfff646daaf16df65fbac47e7 is 50, key is test_row_0/C:col10/1733359400504/Put/seqid=0 2024-12-05T00:43:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:22,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742281_1457 (size=12151) 2024-12-05T00:43:22,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/ee3302bcfff646daaf16df65fbac47e7 2024-12-05T00:43:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5e1a8db3966b4ee8b52e096b21b2d661 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5e1a8db3966b4ee8b52e096b21b2d661 2024-12-05T00:43:22,672 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5e1a8db3966b4ee8b52e096b21b2d661, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T00:43:22,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/01b8afb3c7de4ba195c685894354c378 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/01b8afb3c7de4ba195c685894354c378 2024-12-05T00:43:22,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/01b8afb3c7de4ba195c685894354c378, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T00:43:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/ee3302bcfff646daaf16df65fbac47e7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/ee3302bcfff646daaf16df65fbac47e7 2024-12-05T00:43:22,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/ee3302bcfff646daaf16df65fbac47e7, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T00:43:22,680 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=46.96 KB/48090 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 846ms, sequenceid=210, compaction requested=false 2024-12-05T00:43:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-05T00:43:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-05T00:43:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:22,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:43:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:22,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:22,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-05T00:43:22,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-12-05T00:43:22,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4640 sec 2024-12-05T00:43:22,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/06353715118d43938d79946c75aa1687 is 50, key is test_row_0/A:col10/1733359402677/Put/seqid=0 2024-12-05T00:43:22,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742282_1458 (size=14541) 2024-12-05T00:43:22,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359402709, max timestamp=1733359402727Num uniques 859; 2024-12-05T00:43:22,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:22,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359402709, max timestamp=1733359402727Num uniques 859; 2024-12-05T00:43:22,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:22,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359402709, max timestamp=1733359402727Num uniques 859; 2024-12-05T00:43:22,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359462766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359462766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:22,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359462767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:22,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359462872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359462872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:22,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359462881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359463082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359463082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/06353715118d43938d79946c75aa1687 2024-12-05T00:43:23,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/166185b9e2fe40f0a53c8edcf08634ea is 50, key is test_row_0/B:col10/1733359402677/Put/seqid=0 2024-12-05T00:43:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359463091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742283_1459 (size=12151) 2024-12-05T00:43:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T00:43:23,371 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:23,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-05T00:43:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:23,373 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:23,374 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:23,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:23,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359463391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359463401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359463412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:23,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/166185b9e2fe40f0a53c8edcf08634ea 2024-12-05T00:43:23,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/0722b5be188646e99ea3cd7d1f8311ce is 50, key is test_row_0/C:col10/1733359402677/Put/seqid=0 2024-12-05T00:43:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742284_1460 (size=12151) 2024-12-05T00:43:23,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:23,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359463901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/0722b5be188646e99ea3cd7d1f8311ce 2024-12-05T00:43:23,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/06353715118d43938d79946c75aa1687 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/06353715118d43938d79946c75aa1687 2024-12-05T00:43:23,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/06353715118d43938d79946c75aa1687, entries=200, sequenceid=224, filesize=14.2 K 2024-12-05T00:43:23,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/166185b9e2fe40f0a53c8edcf08634ea as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/166185b9e2fe40f0a53c8edcf08634ea 2024-12-05T00:43:23,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/166185b9e2fe40f0a53c8edcf08634ea, entries=150, sequenceid=224, filesize=11.9 K 2024-12-05T00:43:23,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/0722b5be188646e99ea3cd7d1f8311ce as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0722b5be188646e99ea3cd7d1f8311ce 2024-12-05T00:43:23,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0722b5be188646e99ea3cd7d1f8311ce, entries=150, sequenceid=224, filesize=11.9 K 2024-12-05T00:43:23,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1241ms, sequenceid=224, compaction requested=true 2024-12-05T00:43:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:23,925 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:23,926 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:23,926 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:23,926 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:23,926 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/B is initiating minor compaction (all files) 2024-12-05T00:43:23,926 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/A is initiating minor compaction (all files) 2024-12-05T00:43:23,926 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/B in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,926 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/A in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,926 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0bed1e53ad254a4abb11223c88969f46, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/01b8afb3c7de4ba195c685894354c378, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/166185b9e2fe40f0a53c8edcf08634ea] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=36.0 K 2024-12-05T00:43:23,926 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/24917594092946dfa915c3bc50d0077c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5e1a8db3966b4ee8b52e096b21b2d661, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/06353715118d43938d79946c75aa1687] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=38.3 K 2024-12-05T00:43:23,926 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bed1e53ad254a4abb11223c88969f46, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733359399830 2024-12-05T00:43:23,927 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24917594092946dfa915c3bc50d0077c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733359399830 2024-12-05T00:43:23,927 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b8afb3c7de4ba195c685894354c378, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733359400499 2024-12-05T00:43:23,927 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e1a8db3966b4ee8b52e096b21b2d661, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733359400499 2024-12-05T00:43:23,927 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 166185b9e2fe40f0a53c8edcf08634ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733359402675 2024-12-05T00:43:23,927 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06353715118d43938d79946c75aa1687, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733359402666 2024-12-05T00:43:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:23,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T00:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:23,933 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:23,933 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/fca2106e7ef241cca34107d3b6eac7a3 is 50, key is test_row_0/B:col10/1733359402677/Put/seqid=0 2024-12-05T00:43:23,936 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:23,936 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4fae7a24e58448ab90a711048889dba5 is 50, key is test_row_0/A:col10/1733359402677/Put/seqid=0 2024-12-05T00:43:23,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742285_1461 (size=12629) 2024-12-05T00:43:23,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5d5d56fa8bb64328bab761c2004ab356 is 50, key is test_row_0/A:col10/1733359402766/Put/seqid=0 2024-12-05T00:43:23,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742286_1462 (size=12629) 2024-12-05T00:43:23,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742287_1463 (size=14691) 2024-12-05T00:43:23,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5d5d56fa8bb64328bab761c2004ab356 2024-12-05T00:43:23,960 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/fca2106e7ef241cca34107d3b6eac7a3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/fca2106e7ef241cca34107d3b6eac7a3 2024-12-05T00:43:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359463959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,966 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/B of 1e9fd1cbd3f1490919d61090b95cb9a3 into fca2106e7ef241cca34107d3b6eac7a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:23,966 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:23,966 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/B, priority=13, startTime=1733359403925; duration=0sec 2024-12-05T00:43:23,966 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:23,966 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:B 2024-12-05T00:43:23,966 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:23,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/752c41abc340445a94d109d5b3419e7d is 50, key is test_row_0/B:col10/1733359402766/Put/seqid=0 2024-12-05T00:43:23,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359463964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:23,972 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:23,972 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/C is initiating minor compaction (all files) 2024-12-05T00:43:23,972 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/C in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,972 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/9c00137b1bbb403fb04c6004095a863f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/ee3302bcfff646daaf16df65fbac47e7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0722b5be188646e99ea3cd7d1f8311ce] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=36.0 K 2024-12-05T00:43:23,972 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c00137b1bbb403fb04c6004095a863f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733359399830 2024-12-05T00:43:23,973 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3302bcfff646daaf16df65fbac47e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733359400499 2024-12-05T00:43:23,973 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0722b5be188646e99ea3cd7d1f8311ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733359402675 2024-12-05T00:43:23,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742288_1464 (size=12301) 2024-12-05T00:43:23,981 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:23,981 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63 is 50, key is test_row_0/C:col10/1733359402677/Put/seqid=0 2024-12-05T00:43:23,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:23,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742289_1465 (size=12629) 2024-12-05T00:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:24,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359464071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:24,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359464081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:24,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359464282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:24,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359464291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:24,353 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4fae7a24e58448ab90a711048889dba5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4fae7a24e58448ab90a711048889dba5 2024-12-05T00:43:24,356 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/A of 1e9fd1cbd3f1490919d61090b95cb9a3 into 4fae7a24e58448ab90a711048889dba5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:24,356 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:24,356 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/A, priority=13, startTime=1733359403925; duration=0sec 2024-12-05T00:43:24,356 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:24,356 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:A 2024-12-05T00:43:24,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/752c41abc340445a94d109d5b3419e7d 2024-12-05T00:43:24,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/3b9b921c5b1641df8fb6b7d7e8ad2541 is 50, key is test_row_0/C:col10/1733359402766/Put/seqid=0 2024-12-05T00:43:24,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742290_1466 (size=12301) 2024-12-05T00:43:24,394 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63 2024-12-05T00:43:24,399 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/C of 1e9fd1cbd3f1490919d61090b95cb9a3 into 53dd1a08d55e4da3a9b2cc3a4c0b8d63(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:24,399 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:24,399 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/C, priority=13, startTime=1733359403926; duration=0sec 2024-12-05T00:43:24,399 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:24,399 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:C 2024-12-05T00:43:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:24,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359464592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:24,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359464602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:24,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:24,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/3b9b921c5b1641df8fb6b7d7e8ad2541 2024-12-05T00:43:24,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5d5d56fa8bb64328bab761c2004ab356 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5d5d56fa8bb64328bab761c2004ab356 2024-12-05T00:43:24,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5d5d56fa8bb64328bab761c2004ab356, entries=200, sequenceid=260, filesize=14.3 K 2024-12-05T00:43:24,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/752c41abc340445a94d109d5b3419e7d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/752c41abc340445a94d109d5b3419e7d 2024-12-05T00:43:24,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/752c41abc340445a94d109d5b3419e7d, entries=150, sequenceid=260, filesize=12.0 K 2024-12-05T00:43:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/3b9b921c5b1641df8fb6b7d7e8ad2541 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/3b9b921c5b1641df8fb6b7d7e8ad2541 2024-12-05T00:43:24,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/3b9b921c5b1641df8fb6b7d7e8ad2541, entries=150, sequenceid=260, filesize=12.0 K 2024-12-05T00:43:24,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 879ms, sequenceid=260, compaction requested=false 2024-12-05T00:43:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:24,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:24,900 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/ab86a81ed143455b83fc4fdd4770d1f0 is 50, key is test_row_0/A:col10/1733359403957/Put/seqid=0 2024-12-05T00:43:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742291_1467 (size=12301) 2024-12-05T00:43:24,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:25,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359404950, max timestamp=1733359405002Num uniques 859; 2024-12-05T00:43:25,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359404950, max timestamp=1733359405002Num uniques 859; 2024-12-05T00:43:25,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:43:25,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:43:25,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359404950, max timestamp=1733359405002Num uniques 859; 2024-12-05T00:43:25,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:43:25,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359465095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359465112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359465122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:25,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359465212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:25,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/ab86a81ed143455b83fc4fdd4770d1f0 2024-12-05T00:43:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/82e855de81434dd4bc3a05fbfc31b3f4 is 50, key is test_row_0/B:col10/1733359403957/Put/seqid=0 2024-12-05T00:43:25,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742292_1468 (size=12301) 2024-12-05T00:43:25,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359465432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:25,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/82e855de81434dd4bc3a05fbfc31b3f4 2024-12-05T00:43:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/b884239ca90b4bcd8e78e21027041248 is 50, key is test_row_0/C:col10/1733359403957/Put/seqid=0 2024-12-05T00:43:25,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742293_1469 (size=12301) 2024-12-05T00:43:25,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359465752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,123 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/b884239ca90b4bcd8e78e21027041248 2024-12-05T00:43:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/ab86a81ed143455b83fc4fdd4770d1f0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ab86a81ed143455b83fc4fdd4770d1f0 2024-12-05T00:43:26,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ab86a81ed143455b83fc4fdd4770d1f0, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T00:43:26,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/82e855de81434dd4bc3a05fbfc31b3f4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/82e855de81434dd4bc3a05fbfc31b3f4 2024-12-05T00:43:26,132 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/82e855de81434dd4bc3a05fbfc31b3f4, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T00:43:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/b884239ca90b4bcd8e78e21027041248 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b884239ca90b4bcd8e78e21027041248 2024-12-05T00:43:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359466132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:26,134 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,136 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b884239ca90b4bcd8e78e21027041248, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T00:43:26,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1237ms, sequenceid=273, compaction requested=true 2024-12-05T00:43:26,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:26,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:26,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-05T00:43:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-05T00:43:26,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-05T00:43:26,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7640 sec 2024-12-05T00:43:26,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.7680 sec 2024-12-05T00:43:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:26,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-05T00:43:26,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:26,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T00:43:26,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T00:43:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T00:43:26,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/e491a6f290cf488d87c7558dd96fbc21 is 50, key is test_row_0/A:col10/1733359405057/Put/seqid=0 2024-12-05T00:43:26,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742294_1470 (size=14741) 2024-12-05T00:43:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359466176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359466262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359466291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:26,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359466512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/e491a6f290cf488d87c7558dd96fbc21 2024-12-05T00:43:26,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/9faa423e61bc4cb8866fee6b86ac6917 is 50, key is test_row_0/B:col10/1733359405057/Put/seqid=0 2024-12-05T00:43:26,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742295_1471 (size=12301) 2024-12-05T00:43:26,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/9faa423e61bc4cb8866fee6b86ac6917 2024-12-05T00:43:26,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/cb882c486b064d81b48036052e8c2c86 is 50, key is test_row_0/C:col10/1733359405057/Put/seqid=0 2024-12-05T00:43:26,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742296_1472 (size=12301) 2024-12-05T00:43:26,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359466821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:26,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/cb882c486b064d81b48036052e8c2c86 2024-12-05T00:43:26,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/e491a6f290cf488d87c7558dd96fbc21 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/e491a6f290cf488d87c7558dd96fbc21 2024-12-05T00:43:26,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/e491a6f290cf488d87c7558dd96fbc21, entries=200, sequenceid=311, filesize=14.4 K 2024-12-05T00:43:26,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/9faa423e61bc4cb8866fee6b86ac6917 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9faa423e61bc4cb8866fee6b86ac6917 2024-12-05T00:43:26,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9faa423e61bc4cb8866fee6b86ac6917, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T00:43:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/cb882c486b064d81b48036052e8c2c86 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/cb882c486b064d81b48036052e8c2c86 2024-12-05T00:43:26,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/cb882c486b064d81b48036052e8c2c86, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T00:43:26,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 841ms, sequenceid=311, compaction requested=true 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:26,988 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:26,988 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:26,989 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:26,989 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:26,989 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/B is initiating minor compaction (all files) 2024-12-05T00:43:26,989 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/A is initiating minor compaction (all files) 2024-12-05T00:43:26,989 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/B in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:26,989 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/A in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:26,989 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/fca2106e7ef241cca34107d3b6eac7a3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/752c41abc340445a94d109d5b3419e7d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/82e855de81434dd4bc3a05fbfc31b3f4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9faa423e61bc4cb8866fee6b86ac6917] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=48.4 K 2024-12-05T00:43:26,989 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4fae7a24e58448ab90a711048889dba5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5d5d56fa8bb64328bab761c2004ab356, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ab86a81ed143455b83fc4fdd4770d1f0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/e491a6f290cf488d87c7558dd96fbc21] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=53.1 K 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting fca2106e7ef241cca34107d3b6eac7a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733359402675 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fae7a24e58448ab90a711048889dba5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733359402675 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 752c41abc340445a94d109d5b3419e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733359402765 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d5d56fa8bb64328bab761c2004ab356, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733359402759 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab86a81ed143455b83fc4fdd4770d1f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733359403941 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e855de81434dd4bc3a05fbfc31b3f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733359403941 2024-12-05T00:43:26,990 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 9faa423e61bc4cb8866fee6b86ac6917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733359405057 2024-12-05T00:43:26,991 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e491a6f290cf488d87c7558dd96fbc21, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733359405057 2024-12-05T00:43:27,006 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:27,006 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:27,007 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/2da6f16631654107b7b1330bfd0a8ecd is 50, key is test_row_0/B:col10/1733359405057/Put/seqid=0 2024-12-05T00:43:27,007 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/447b6b3408bf4732a4dbf63155af76e3 is 50, key is test_row_0/A:col10/1733359405057/Put/seqid=0 2024-12-05T00:43:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742298_1474 (size=12915) 2024-12-05T00:43:27,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742297_1473 (size=12915) 2024-12-05T00:43:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:27,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:27,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/532298c8c9b0461692ba5914018f1643 is 50, key is test_row_0/A:col10/1733359406173/Put/seqid=0 2024-12-05T00:43:27,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742299_1475 (size=14741) 2024-12-05T00:43:27,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359407374, max timestamp=1733359407399Num uniques 859; 2024-12-05T00:43:27,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359407374, max timestamp=1733359407399Num uniques 859; 2024-12-05T00:43:27,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:27,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:27,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359407374, max timestamp=1733359407399Num uniques 859; 2024-12-05T00:43:27,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:27,425 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/2da6f16631654107b7b1330bfd0a8ecd as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/2da6f16631654107b7b1330bfd0a8ecd 2024-12-05T00:43:27,425 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/447b6b3408bf4732a4dbf63155af76e3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/447b6b3408bf4732a4dbf63155af76e3 2024-12-05T00:43:27,428 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/A of 1e9fd1cbd3f1490919d61090b95cb9a3 into 447b6b3408bf4732a4dbf63155af76e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:27,428 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/B of 1e9fd1cbd3f1490919d61090b95cb9a3 into 2da6f16631654107b7b1330bfd0a8ecd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:27,429 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/A, priority=12, startTime=1733359406988; duration=0sec 2024-12-05T00:43:27,429 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/B, priority=12, startTime=1733359406988; duration=0sec 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:A 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:B 2024-12-05T00:43:27,429 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:27,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:27,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/C is initiating minor compaction (all files) 2024-12-05T00:43:27,430 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/C in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:27,430 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/3b9b921c5b1641df8fb6b7d7e8ad2541, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b884239ca90b4bcd8e78e21027041248, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/cb882c486b064d81b48036052e8c2c86] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=48.4 K 2024-12-05T00:43:27,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53dd1a08d55e4da3a9b2cc3a4c0b8d63, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733359402675 2024-12-05T00:43:27,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b9b921c5b1641df8fb6b7d7e8ad2541, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733359402765 2024-12-05T00:43:27,430 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting b884239ca90b4bcd8e78e21027041248, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733359403941 2024-12-05T00:43:27,431 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb882c486b064d81b48036052e8c2c86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733359405057 2024-12-05T00:43:27,437 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:27,437 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/63acc4c9de864b1c83b3c3e4781fd44b is 50, key is test_row_0/C:col10/1733359405057/Put/seqid=0 2024-12-05T00:43:27,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742300_1476 (size=12915) 2024-12-05T00:43:27,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:27,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359467468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359467469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T00:43:27,542 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:27,542 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-05T00:43:27,544 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:27,545 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:27,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359467581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359467581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:27,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/532298c8c9b0461692ba5914018f1643 2024-12-05T00:43:27,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:27,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:27,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/ebbc6a99244646a8915865139c4b98f2 is 50, key is test_row_0/B:col10/1733359406173/Put/seqid=0 2024-12-05T00:43:27,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742301_1477 (size=12301) 2024-12-05T00:43:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359467791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359467791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:27,846 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/63acc4c9de864b1c83b3c3e4781fd44b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/63acc4c9de864b1c83b3c3e4781fd44b 2024-12-05T00:43:27,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:27,850 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/C of 1e9fd1cbd3f1490919d61090b95cb9a3 into 63acc4c9de864b1c83b3c3e4781fd44b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:27,850 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:27,850 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/C, priority=12, startTime=1733359406988; duration=0sec 2024-12-05T00:43:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:27,850 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:27,850 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:C 2024-12-05T00:43:27,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:28,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/ebbc6a99244646a8915865139c4b98f2 2024-12-05T00:43:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359468102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359468102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/c8b2053f6dd64e78862f7acad3cfa670 is 50, key is test_row_0/C:col10/1733359406173/Put/seqid=0 2024-12-05T00:43:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742302_1478 (size=12301) 2024-12-05T00:43:28,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53014 deadline: 1733359468151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,154 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 1e9fd1cbd3f1490919d61090b95cb9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:28,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:28,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:28,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:28,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/c8b2053f6dd64e78862f7acad3cfa670 2024-12-05T00:43:28,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/532298c8c9b0461692ba5914018f1643 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/532298c8c9b0461692ba5914018f1643 2024-12-05T00:43:28,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/532298c8c9b0461692ba5914018f1643, entries=200, sequenceid=321, filesize=14.4 K 2024-12-05T00:43:28,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/ebbc6a99244646a8915865139c4b98f2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/ebbc6a99244646a8915865139c4b98f2 2024-12-05T00:43:28,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/ebbc6a99244646a8915865139c4b98f2, entries=150, sequenceid=321, filesize=12.0 K 2024-12-05T00:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/c8b2053f6dd64e78862f7acad3cfa670 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/c8b2053f6dd64e78862f7acad3cfa670 2024-12-05T00:43:28,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/c8b2053f6dd64e78862f7acad3cfa670, entries=150, sequenceid=321, filesize=12.0 K 2024-12-05T00:43:28,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1245ms, sequenceid=321, compaction requested=false 2024-12-05T00:43:28,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:28,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:28,613 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. as already flushing 2024-12-05T00:43:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/52515e9fd80047faae7cac4f0ba9fe72 is 50, key is test_row_0/A:col10/1733359407466/Put/seqid=0 2024-12-05T00:43:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742303_1479 (size=12301) 2024-12-05T00:43:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359468638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359468645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:28,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359468751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359468761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359468961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359468971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:29,021 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/52515e9fd80047faae7cac4f0ba9fe72 2024-12-05T00:43:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/1ea1e3caa7c34c17b6fe2521ce82eebd is 50, key is test_row_0/B:col10/1733359407466/Put/seqid=0 2024-12-05T00:43:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742304_1480 (size=12301) 2024-12-05T00:43:29,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359469272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:29,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359469281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:29,430 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/1ea1e3caa7c34c17b6fe2521ce82eebd 2024-12-05T00:43:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/5f42b5a48a04449dbc4c7c1297c245e3 is 50, key is test_row_0/C:col10/1733359407466/Put/seqid=0 2024-12-05T00:43:29,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742305_1481 (size=12301) 2024-12-05T00:43:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1733359469782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:29,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53054 deadline: 1733359469801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 , the old value is region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=2 because the exception is null or not the one we care about 2024-12-05T00:43:29,840 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/5f42b5a48a04449dbc4c7c1297c245e3 2024-12-05T00:43:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/52515e9fd80047faae7cac4f0ba9fe72 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/52515e9fd80047faae7cac4f0ba9fe72 2024-12-05T00:43:29,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/52515e9fd80047faae7cac4f0ba9fe72, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T00:43:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/1ea1e3caa7c34c17b6fe2521ce82eebd as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/1ea1e3caa7c34c17b6fe2521ce82eebd 2024-12-05T00:43:29,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/1ea1e3caa7c34c17b6fe2521ce82eebd, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T00:43:29,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/5f42b5a48a04449dbc4c7c1297c245e3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/5f42b5a48a04449dbc4c7c1297c245e3 2024-12-05T00:43:29,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/5f42b5a48a04449dbc4c7c1297c245e3, entries=150, sequenceid=360, filesize=12.0 K 2024-12-05T00:43:29,852 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1239ms, sequenceid=360, compaction requested=true 2024-12-05T00:43:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-05T00:43:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-05T00:43:29,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-05T00:43:29,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3080 sec 2024-12-05T00:43:29,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.3140 sec 2024-12-05T00:43:30,602 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-12-05T00:43:30,602 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,602 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,602 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:30,603 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-12-05T00:43:30,603 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,603 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,603 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:30,604 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-12-05T00:43:30,604 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,604 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,604 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:30,606 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-12-05T00:43:30,606 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,606 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,606 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:30,608 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-12-05T00:43:30,608 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,608 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,608 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:30,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:43:30,808 INFO [Thread-1894 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1894. 2024-12-05T00:43:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:30,808 DEBUG [Thread-1894 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,808 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:30,808 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:30,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:30,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/90263414422d46858416cc7af042f43f is 50, key is test_row_0/A:col10/1733359408637/Put/seqid=0 2024-12-05T00:43:30,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742306_1482 (size=12301) 2024-12-05T00:43:30,823 INFO [Thread-1892 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1892. 2024-12-05T00:43:30,823 DEBUG [Thread-1892 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:30,823 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,823 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:30,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:31,215 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-12-05T00:43:31,215 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:31,215 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:31,216 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:31,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:31,245 INFO [Thread-1896 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1896. 2024-12-05T00:43:31,245 DEBUG [Thread-1896 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:31,245 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:31,246 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:31,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:31,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/90263414422d46858416cc7af042f43f 2024-12-05T00:43:31,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/945af498bf5344aebd4dbd856b401e42 is 50, key is test_row_0/B:col10/1733359408637/Put/seqid=0 2024-12-05T00:43:31,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742307_1483 (size=12301) 2024-12-05T00:43:31,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/945af498bf5344aebd4dbd856b401e42 2024-12-05T00:43:31,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/18585f30fef143588574b6111927d232 is 50, key is test_row_0/C:col10/1733359408637/Put/seqid=0 2024-12-05T00:43:31,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742308_1484 (size=12301) 2024-12-05T00:43:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T00:43:31,701 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:32,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/18585f30fef143588574b6111927d232 2024-12-05T00:43:32,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/90263414422d46858416cc7af042f43f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/90263414422d46858416cc7af042f43f 2024-12-05T00:43:32,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/90263414422d46858416cc7af042f43f, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T00:43:32,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/945af498bf5344aebd4dbd856b401e42 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/945af498bf5344aebd4dbd856b401e42 2024-12-05T00:43:32,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/945af498bf5344aebd4dbd856b401e42, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T00:43:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/18585f30fef143588574b6111927d232 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/18585f30fef143588574b6111927d232 2024-12-05T00:43:32,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/18585f30fef143588574b6111927d232, entries=150, sequenceid=371, filesize=12.0 K 2024-12-05T00:43:32,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1309ms, sequenceid=371, compaction requested=true 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e9fd1cbd3f1490919d61090b95cb9a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:32,116 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:32,116 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:32,117 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:32,117 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/A is initiating minor compaction (all files) 2024-12-05T00:43:32,117 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:32,118 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/A in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 1e9fd1cbd3f1490919d61090b95cb9a3/B is initiating minor compaction (all files) 2024-12-05T00:43:32,118 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e9fd1cbd3f1490919d61090b95cb9a3/B in TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:32,118 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/447b6b3408bf4732a4dbf63155af76e3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/532298c8c9b0461692ba5914018f1643, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/52515e9fd80047faae7cac4f0ba9fe72, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/90263414422d46858416cc7af042f43f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=51.0 K 2024-12-05T00:43:32,118 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/2da6f16631654107b7b1330bfd0a8ecd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/ebbc6a99244646a8915865139c4b98f2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/1ea1e3caa7c34c17b6fe2521ce82eebd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/945af498bf5344aebd4dbd856b401e42] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp, totalSize=48.7 K 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da6f16631654107b7b1330bfd0a8ecd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733359405057 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 447b6b3408bf4732a4dbf63155af76e3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733359405057 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebbc6a99244646a8915865139c4b98f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733359406173 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 532298c8c9b0461692ba5914018f1643, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733359406166 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea1e3caa7c34c17b6fe2521ce82eebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733359407447 2024-12-05T00:43:32,118 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 52515e9fd80047faae7cac4f0ba9fe72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733359407447 2024-12-05T00:43:32,119 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 945af498bf5344aebd4dbd856b401e42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733359408625 2024-12-05T00:43:32,119 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 90263414422d46858416cc7af042f43f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733359408625 2024-12-05T00:43:32,124 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:32,125 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/44a8f43fdd7c4b638b8b14fa7c9a5d7a is 50, key is test_row_0/B:col10/1733359408637/Put/seqid=0 2024-12-05T00:43:32,125 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e9fd1cbd3f1490919d61090b95cb9a3#A#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:32,126 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4e2c543956534016932dc1639e075a2c is 50, key is test_row_0/A:col10/1733359408637/Put/seqid=0 2024-12-05T00:43:32,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742309_1485 (size=13051) 2024-12-05T00:43:32,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742310_1486 (size=13051) 2024-12-05T00:43:32,174 INFO [Thread-1890 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1890. 2024-12-05T00:43:32,174 DEBUG [Thread-1890 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:32,174 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:32,174 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:32,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1683 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5042 rows 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1675 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5018 rows 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1671 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5010 rows 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1683 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5048 rows 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1683 2024-12-05T00:43:32,175 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5049 rows 2024-12-05T00:43:32,175 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:43:32,175 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:43:32,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:32,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:32,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:32,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T00:43:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T00:43:32,183 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359412183"}]},"ts":"1733359412183"} 2024-12-05T00:43:32,185 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T00:43:32,185 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T00:43:32,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:43:32,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, UNASSIGN}] 2024-12-05T00:43:32,188 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, UNASSIGN 2024-12-05T00:43:32,189 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1e9fd1cbd3f1490919d61090b95cb9a3, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:32,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, UNASSIGN because future has completed 2024-12-05T00:43:32,191 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:43:32,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:43:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T00:43:32,344 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:32,345 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:43:32,345 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 1e9fd1cbd3f1490919d61090b95cb9a3, disabling compactions & flushes 2024-12-05T00:43:32,345 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T00:43:32,539 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/44a8f43fdd7c4b638b8b14fa7c9a5d7a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/44a8f43fdd7c4b638b8b14fa7c9a5d7a 2024-12-05T00:43:32,539 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/4e2c543956534016932dc1639e075a2c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4e2c543956534016932dc1639e075a2c 2024-12-05T00:43:32,545 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/A of 1e9fd1cbd3f1490919d61090b95cb9a3 into 4e2c543956534016932dc1639e075a2c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:32,545 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e9fd1cbd3f1490919d61090b95cb9a3/B of 1e9fd1cbd3f1490919d61090b95cb9a3 into 44a8f43fdd7c4b638b8b14fa7c9a5d7a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e9fd1cbd3f1490919d61090b95cb9a3: 2024-12-05T00:43:32,546 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/A, priority=12, startTime=1733359412116; duration=0sec 2024-12-05T00:43:32,546 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., storeName=1e9fd1cbd3f1490919d61090b95cb9a3/B, priority=12, startTime=1733359412116; duration=0sec 2024-12-05T00:43:32,546 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:B 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:A 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. because compaction request was cancelled 2024-12-05T00:43:32,546 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e9fd1cbd3f1490919d61090b95cb9a3:C 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. after waiting 0 ms 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:32,546 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 1e9fd1cbd3f1490919d61090b95cb9a3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=A 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=B 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e9fd1cbd3f1490919d61090b95cb9a3, store=C 2024-12-05T00:43:32,546 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:32,550 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5b2972960f5842beb10b7de1e4bf4ea8 is 50, key is test_row_0/A:col10/1733359412172/Put/seqid=0 2024-12-05T00:43:32,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742311_1487 (size=12301) 2024-12-05T00:43:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T00:43:32,955 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5b2972960f5842beb10b7de1e4bf4ea8 2024-12-05T00:43:32,966 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/52fa7db614504e428552d9d73fe8f709 is 50, key is test_row_0/B:col10/1733359412172/Put/seqid=0 2024-12-05T00:43:32,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742312_1488 (size=12301) 2024-12-05T00:43:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T00:43:33,371 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/52fa7db614504e428552d9d73fe8f709 2024-12-05T00:43:33,385 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/34b446ab58564667a4cda923cb52d4a6 is 50, key is test_row_0/C:col10/1733359412172/Put/seqid=0 2024-12-05T00:43:33,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742313_1489 (size=12301) 2024-12-05T00:43:33,790 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/34b446ab58564667a4cda923cb52d4a6 2024-12-05T00:43:33,796 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/A/5b2972960f5842beb10b7de1e4bf4ea8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5b2972960f5842beb10b7de1e4bf4ea8 2024-12-05T00:43:33,800 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5b2972960f5842beb10b7de1e4bf4ea8, entries=150, sequenceid=380, filesize=12.0 K 2024-12-05T00:43:33,801 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/B/52fa7db614504e428552d9d73fe8f709 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/52fa7db614504e428552d9d73fe8f709 2024-12-05T00:43:33,805 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/52fa7db614504e428552d9d73fe8f709, entries=150, sequenceid=380, filesize=12.0 K 2024-12-05T00:43:33,806 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/.tmp/C/34b446ab58564667a4cda923cb52d4a6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/34b446ab58564667a4cda923cb52d4a6 2024-12-05T00:43:33,810 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/34b446ab58564667a4cda923cb52d4a6, entries=150, sequenceid=380, filesize=12.0 K 2024-12-05T00:43:33,810 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1264ms, sequenceid=380, compaction requested=true 2024-12-05T00:43:33,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/69c1f6810121462ea77d2e462fc5afe3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/656c4aaacde24994939b24db7a22d9a4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4064bc1131554e41b142d64aa620f6f3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/9e3e1839c84a47d98911049f6675fe02, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/1f6854cb0d8d40728534ed6799a4dc1c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/a86850cab2934ad8b0348c8118bb0931, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ac906667e4ef4290af27f1822d21f351, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/7f3d274b942f49958fa81719c43c3834, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/54606e2d3d4d46408a97e5a647392cda, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/8d1064c7277045cba3f12615631bd125, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/fc252732482d45fa98dcfda24a08bbf4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/24917594092946dfa915c3bc50d0077c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5e1a8db3966b4ee8b52e096b21b2d661, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/06353715118d43938d79946c75aa1687, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4fae7a24e58448ab90a711048889dba5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5d5d56fa8bb64328bab761c2004ab356, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ab86a81ed143455b83fc4fdd4770d1f0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/e491a6f290cf488d87c7558dd96fbc21, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/447b6b3408bf4732a4dbf63155af76e3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/532298c8c9b0461692ba5914018f1643, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/52515e9fd80047faae7cac4f0ba9fe72, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/90263414422d46858416cc7af042f43f] to archive 2024-12-05T00:43:33,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:43:33,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/69c1f6810121462ea77d2e462fc5afe3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/69c1f6810121462ea77d2e462fc5afe3 2024-12-05T00:43:33,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/656c4aaacde24994939b24db7a22d9a4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/656c4aaacde24994939b24db7a22d9a4 2024-12-05T00:43:33,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4064bc1131554e41b142d64aa620f6f3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4064bc1131554e41b142d64aa620f6f3 2024-12-05T00:43:33,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/9e3e1839c84a47d98911049f6675fe02 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/9e3e1839c84a47d98911049f6675fe02 2024-12-05T00:43:33,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/1f6854cb0d8d40728534ed6799a4dc1c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/1f6854cb0d8d40728534ed6799a4dc1c 2024-12-05T00:43:33,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/a86850cab2934ad8b0348c8118bb0931 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/a86850cab2934ad8b0348c8118bb0931 2024-12-05T00:43:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ac906667e4ef4290af27f1822d21f351 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ac906667e4ef4290af27f1822d21f351 2024-12-05T00:43:33,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/7f3d274b942f49958fa81719c43c3834 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/7f3d274b942f49958fa81719c43c3834 2024-12-05T00:43:33,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/54606e2d3d4d46408a97e5a647392cda to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/54606e2d3d4d46408a97e5a647392cda 2024-12-05T00:43:33,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/8d1064c7277045cba3f12615631bd125 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/8d1064c7277045cba3f12615631bd125 2024-12-05T00:43:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/fc252732482d45fa98dcfda24a08bbf4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/fc252732482d45fa98dcfda24a08bbf4 2024-12-05T00:43:33,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/24917594092946dfa915c3bc50d0077c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/24917594092946dfa915c3bc50d0077c 2024-12-05T00:43:33,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5e1a8db3966b4ee8b52e096b21b2d661 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5e1a8db3966b4ee8b52e096b21b2d661 2024-12-05T00:43:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/06353715118d43938d79946c75aa1687 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/06353715118d43938d79946c75aa1687 2024-12-05T00:43:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4fae7a24e58448ab90a711048889dba5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4fae7a24e58448ab90a711048889dba5 2024-12-05T00:43:33,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5d5d56fa8bb64328bab761c2004ab356 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5d5d56fa8bb64328bab761c2004ab356 2024-12-05T00:43:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ab86a81ed143455b83fc4fdd4770d1f0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/ab86a81ed143455b83fc4fdd4770d1f0 2024-12-05T00:43:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/e491a6f290cf488d87c7558dd96fbc21 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/e491a6f290cf488d87c7558dd96fbc21 2024-12-05T00:43:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/447b6b3408bf4732a4dbf63155af76e3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/447b6b3408bf4732a4dbf63155af76e3 2024-12-05T00:43:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/532298c8c9b0461692ba5914018f1643 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/532298c8c9b0461692ba5914018f1643 2024-12-05T00:43:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/52515e9fd80047faae7cac4f0ba9fe72 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/52515e9fd80047faae7cac4f0ba9fe72 2024-12-05T00:43:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/90263414422d46858416cc7af042f43f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/90263414422d46858416cc7af042f43f 2024-12-05T00:43:33,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9240ddb30a5b44ae8831bc9b44595e16, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/b40711c52400493dbf832cae95177321, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/3455dbffa2884b82802d3e4cbe4cc162, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/afd4f758845f4fbdafe35364706e161b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/182caa79e13c4fcb9b0371d4de9da238, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/e7c3cf49fa9b41cbba5178d942226947, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/eeb29d44b72c47029677e3fa95f04467, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/8f531bf3ae094073a6a42b09c38891a1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/37fa0e2981c346738e85c1d90711eaf0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0aa4769d8f834ad2a5a43fa2ebea1975, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0bed1e53ad254a4abb11223c88969f46, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/6023af602467480eb45451c83a1961e0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/01b8afb3c7de4ba195c685894354c378, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/fca2106e7ef241cca34107d3b6eac7a3, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/166185b9e2fe40f0a53c8edcf08634ea, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/752c41abc340445a94d109d5b3419e7d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/82e855de81434dd4bc3a05fbfc31b3f4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/2da6f16631654107b7b1330bfd0a8ecd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9faa423e61bc4cb8866fee6b86ac6917, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/ebbc6a99244646a8915865139c4b98f2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/1ea1e3caa7c34c17b6fe2521ce82eebd, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/945af498bf5344aebd4dbd856b401e42] to archive 2024-12-05T00:43:33,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:43:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9240ddb30a5b44ae8831bc9b44595e16 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9240ddb30a5b44ae8831bc9b44595e16 2024-12-05T00:43:33,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/b40711c52400493dbf832cae95177321 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/b40711c52400493dbf832cae95177321 2024-12-05T00:43:33,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/3455dbffa2884b82802d3e4cbe4cc162 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/3455dbffa2884b82802d3e4cbe4cc162 2024-12-05T00:43:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/afd4f758845f4fbdafe35364706e161b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/afd4f758845f4fbdafe35364706e161b 2024-12-05T00:43:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/182caa79e13c4fcb9b0371d4de9da238 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/182caa79e13c4fcb9b0371d4de9da238 2024-12-05T00:43:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/e7c3cf49fa9b41cbba5178d942226947 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/e7c3cf49fa9b41cbba5178d942226947 2024-12-05T00:43:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/eeb29d44b72c47029677e3fa95f04467 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/eeb29d44b72c47029677e3fa95f04467 2024-12-05T00:43:33,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/8f531bf3ae094073a6a42b09c38891a1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/8f531bf3ae094073a6a42b09c38891a1 2024-12-05T00:43:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/37fa0e2981c346738e85c1d90711eaf0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/37fa0e2981c346738e85c1d90711eaf0 2024-12-05T00:43:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0aa4769d8f834ad2a5a43fa2ebea1975 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0aa4769d8f834ad2a5a43fa2ebea1975 2024-12-05T00:43:33,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0bed1e53ad254a4abb11223c88969f46 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/0bed1e53ad254a4abb11223c88969f46 2024-12-05T00:43:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/6023af602467480eb45451c83a1961e0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/6023af602467480eb45451c83a1961e0 2024-12-05T00:43:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/01b8afb3c7de4ba195c685894354c378 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/01b8afb3c7de4ba195c685894354c378 2024-12-05T00:43:33,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/fca2106e7ef241cca34107d3b6eac7a3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/fca2106e7ef241cca34107d3b6eac7a3 2024-12-05T00:43:33,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/166185b9e2fe40f0a53c8edcf08634ea to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/166185b9e2fe40f0a53c8edcf08634ea 2024-12-05T00:43:33,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/752c41abc340445a94d109d5b3419e7d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/752c41abc340445a94d109d5b3419e7d 2024-12-05T00:43:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/82e855de81434dd4bc3a05fbfc31b3f4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/82e855de81434dd4bc3a05fbfc31b3f4 2024-12-05T00:43:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/2da6f16631654107b7b1330bfd0a8ecd to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/2da6f16631654107b7b1330bfd0a8ecd 2024-12-05T00:43:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9faa423e61bc4cb8866fee6b86ac6917 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/9faa423e61bc4cb8866fee6b86ac6917 2024-12-05T00:43:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/ebbc6a99244646a8915865139c4b98f2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/ebbc6a99244646a8915865139c4b98f2 2024-12-05T00:43:33,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/1ea1e3caa7c34c17b6fe2521ce82eebd to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/1ea1e3caa7c34c17b6fe2521ce82eebd 2024-12-05T00:43:33,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/945af498bf5344aebd4dbd856b401e42 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/945af498bf5344aebd4dbd856b401e42 2024-12-05T00:43:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/7b883a999fce46a99f886cfed82372c1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/508dac27af644762b588ee8453c660ab, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/af3de24282bb401cbe03ba3c04352800, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/4a2a80ee79934f4098c0b7e80c72ec4c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/23de92e861fa4580a9c88a0cb0a60eaf, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0151032053b14e25967af591497e4475, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/2c21f3c7db1f4d3ca4c8efc8972edb55, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/77210a3787cd421e89917aea5862039f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b8ff31623ed443eb814c1f1eb9778f2b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/eec84de374ec4174a53dcc3a6765eacb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/9c00137b1bbb403fb04c6004095a863f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/6c8253d58ef04fff94af23eccc77954a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/ee3302bcfff646daaf16df65fbac47e7, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0722b5be188646e99ea3cd7d1f8311ce, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/3b9b921c5b1641df8fb6b7d7e8ad2541, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b884239ca90b4bcd8e78e21027041248, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/cb882c486b064d81b48036052e8c2c86] to archive 2024-12-05T00:43:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:43:33,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/7b883a999fce46a99f886cfed82372c1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/7b883a999fce46a99f886cfed82372c1 2024-12-05T00:43:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/508dac27af644762b588ee8453c660ab to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/508dac27af644762b588ee8453c660ab 2024-12-05T00:43:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/af3de24282bb401cbe03ba3c04352800 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/af3de24282bb401cbe03ba3c04352800 2024-12-05T00:43:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/4a2a80ee79934f4098c0b7e80c72ec4c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/4a2a80ee79934f4098c0b7e80c72ec4c 2024-12-05T00:43:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/23de92e861fa4580a9c88a0cb0a60eaf to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/23de92e861fa4580a9c88a0cb0a60eaf 2024-12-05T00:43:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0151032053b14e25967af591497e4475 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0151032053b14e25967af591497e4475 2024-12-05T00:43:33,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/2c21f3c7db1f4d3ca4c8efc8972edb55 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/2c21f3c7db1f4d3ca4c8efc8972edb55 2024-12-05T00:43:33,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/77210a3787cd421e89917aea5862039f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/77210a3787cd421e89917aea5862039f 2024-12-05T00:43:33,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b8ff31623ed443eb814c1f1eb9778f2b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b8ff31623ed443eb814c1f1eb9778f2b 2024-12-05T00:43:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/eec84de374ec4174a53dcc3a6765eacb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/eec84de374ec4174a53dcc3a6765eacb 2024-12-05T00:43:33,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/9c00137b1bbb403fb04c6004095a863f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/9c00137b1bbb403fb04c6004095a863f 2024-12-05T00:43:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/6c8253d58ef04fff94af23eccc77954a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/6c8253d58ef04fff94af23eccc77954a 2024-12-05T00:43:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/ee3302bcfff646daaf16df65fbac47e7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/ee3302bcfff646daaf16df65fbac47e7 2024-12-05T00:43:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/53dd1a08d55e4da3a9b2cc3a4c0b8d63 2024-12-05T00:43:33,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0722b5be188646e99ea3cd7d1f8311ce to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/0722b5be188646e99ea3cd7d1f8311ce 2024-12-05T00:43:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/3b9b921c5b1641df8fb6b7d7e8ad2541 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/3b9b921c5b1641df8fb6b7d7e8ad2541 2024-12-05T00:43:33,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b884239ca90b4bcd8e78e21027041248 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/b884239ca90b4bcd8e78e21027041248 2024-12-05T00:43:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/cb882c486b064d81b48036052e8c2c86 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/cb882c486b064d81b48036052e8c2c86 2024-12-05T00:43:33,876 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/recovered.edits/383.seqid, newMaxSeqId=383, maxSeqId=1 2024-12-05T00:43:33,876 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. 2024-12-05T00:43:33,876 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 1e9fd1cbd3f1490919d61090b95cb9a3: Waiting for close lock at 1733359412345Running coprocessor pre-close hooks at 1733359412345Disabling compacts and flushes for region at 1733359412345Disabling writes for close at 1733359412546 (+201 ms)Obtaining lock to block concurrent updates at 1733359412546Preparing flush snapshotting stores in 1e9fd1cbd3f1490919d61090b95cb9a3 at 1733359412546Finished memstore snapshotting TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733359412546Flushing stores of TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3. at 1733359412547 (+1 ms)Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/A: creating writer at 1733359412547Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/A: appending metadata at 1733359412549 (+2 ms)Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/A: closing flushed file at 1733359412549Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/B: creating writer at 1733359412963 (+414 ms)Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/B: appending metadata at 1733359412966 (+3 ms)Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/B: closing flushed file at 1733359412966Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/C: creating writer at 1733359413380 (+414 ms)Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/C: appending metadata at 1733359413385 (+5 ms)Flushing 1e9fd1cbd3f1490919d61090b95cb9a3/C: closing flushed file at 1733359413385Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36669874: reopening flushed file at 1733359413795 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1063d7: reopening flushed file at 1733359413801 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69f6196a: reopening flushed file at 1733359413805 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1e9fd1cbd3f1490919d61090b95cb9a3 in 1264ms, sequenceid=380, compaction requested=true at 1733359413810 (+5 ms)Writing region close event to WAL at 1733359413873 (+63 ms)Running coprocessor post-close hooks at 1733359413876 (+3 ms)Closed at 1733359413876 2024-12-05T00:43:33,878 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:33,879 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1e9fd1cbd3f1490919d61090b95cb9a3, regionState=CLOSED 2024-12-05T00:43:33,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:43:33,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-05T00:43:33,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e9fd1cbd3f1490919d61090b95cb9a3, server=fea72ea5c4b6,42683,1733359267474 in 1.6900 sec 2024-12-05T00:43:33,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-05T00:43:33,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e9fd1cbd3f1490919d61090b95cb9a3, UNASSIGN in 1.6970 sec 2024-12-05T00:43:33,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T00:43:33,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7010 sec 2024-12-05T00:43:33,891 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359413890"}]},"ts":"1733359413890"} 2024-12-05T00:43:33,892 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T00:43:33,892 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T00:43:33,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7160 sec 2024-12-05T00:43:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-05T00:43:34,343 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:34,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T00:43:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T00:43:34,348 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,352 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:34,355 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/recovered.edits] 2024-12-05T00:43:34,359 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4e2c543956534016932dc1639e075a2c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/4e2c543956534016932dc1639e075a2c 2024-12-05T00:43:34,361 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5b2972960f5842beb10b7de1e4bf4ea8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/A/5b2972960f5842beb10b7de1e4bf4ea8 2024-12-05T00:43:34,364 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/44a8f43fdd7c4b638b8b14fa7c9a5d7a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/44a8f43fdd7c4b638b8b14fa7c9a5d7a 2024-12-05T00:43:34,365 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/52fa7db614504e428552d9d73fe8f709 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/B/52fa7db614504e428552d9d73fe8f709 2024-12-05T00:43:34,367 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/18585f30fef143588574b6111927d232 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/18585f30fef143588574b6111927d232 2024-12-05T00:43:34,368 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/34b446ab58564667a4cda923cb52d4a6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/34b446ab58564667a4cda923cb52d4a6 2024-12-05T00:43:34,369 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/5f42b5a48a04449dbc4c7c1297c245e3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/5f42b5a48a04449dbc4c7c1297c245e3 2024-12-05T00:43:34,369 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/63acc4c9de864b1c83b3c3e4781fd44b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/63acc4c9de864b1c83b3c3e4781fd44b 2024-12-05T00:43:34,370 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/c8b2053f6dd64e78862f7acad3cfa670 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/C/c8b2053f6dd64e78862f7acad3cfa670 2024-12-05T00:43:34,371 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/recovered.edits/383.seqid to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3/recovered.edits/383.seqid 2024-12-05T00:43:34,372 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/1e9fd1cbd3f1490919d61090b95cb9a3 2024-12-05T00:43:34,372 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T00:43:34,374 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,375 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T00:43:34,377 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T00:43:34,377 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,378 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T00:43:34,378 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733359414378"}]},"ts":"9223372036854775807"} 2024-12-05T00:43:34,379 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T00:43:34,379 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1e9fd1cbd3f1490919d61090b95cb9a3, NAME => 'TestAcidGuarantees,,1733359388408.1e9fd1cbd3f1490919d61090b95cb9a3.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T00:43:34,379 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T00:43:34,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733359414379"}]},"ts":"9223372036854775807"} 2024-12-05T00:43:34,381 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T00:43:34,381 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-12-05T00:43:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-05T00:43:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T00:43:34,456 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:34,467 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=451 (was 433) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9325 (was 9334) 2024-12-05T00:43:34,478 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=451, ProcessCount=11, AvailableMemoryMB=9325 2024-12-05T00:43:34,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:43:34,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:43:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:34,481 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T00:43:34,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-12-05T00:43:34,481 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T00:43:34,482 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T00:43:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742314_1490 (size=1061) 2024-12-05T00:43:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T00:43:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T00:43:34,892 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18 2024-12-05T00:43:34,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742315_1491 (size=53) 2024-12-05T00:43:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T00:43:35,305 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:43:35,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6544d6803c2ff354880c0eae0e049fdc, disabling compactions & flushes 2024-12-05T00:43:35,306 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. after waiting 0 ms 2024-12-05T00:43:35,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,306 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6544d6803c2ff354880c0eae0e049fdc: Waiting for close lock at 1733359415306Disabling compacts and flushes for region at 1733359415306Disabling writes for close at 1733359415306Writing region close event to WAL at 1733359415306Closed at 1733359415306 2024-12-05T00:43:35,310 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T00:43:35,310 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733359415310"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733359415310"}]},"ts":"1733359415310"} 2024-12-05T00:43:35,313 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T00:43:35,315 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T00:43:35,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359415315"}]},"ts":"1733359415315"} 2024-12-05T00:43:35,317 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T00:43:35,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, ASSIGN}] 2024-12-05T00:43:35,319 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, ASSIGN 2024-12-05T00:43:35,320 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, ASSIGN; state=OFFLINE, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=false 2024-12-05T00:43:35,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:35,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, ASSIGN because future has completed 2024-12-05T00:43:35,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:43:35,591 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T00:43:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T00:43:35,635 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,636 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:43:35,636 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,636 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:43:35,636 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,636 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,638 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,639 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:35,640 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6544d6803c2ff354880c0eae0e049fdc columnFamilyName A 2024-12-05T00:43:35,640 DEBUG [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:35,640 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(327): Store=6544d6803c2ff354880c0eae0e049fdc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:35,640 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,642 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:35,642 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6544d6803c2ff354880c0eae0e049fdc columnFamilyName B 2024-12-05T00:43:35,642 DEBUG [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:35,642 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(327): Store=6544d6803c2ff354880c0eae0e049fdc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:35,643 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,644 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:35,644 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6544d6803c2ff354880c0eae0e049fdc columnFamilyName C 2024-12-05T00:43:35,644 DEBUG [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:35,645 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(327): Store=6544d6803c2ff354880c0eae0e049fdc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:35,645 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,645 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,646 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,646 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,647 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,647 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,648 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:43:35,649 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,651 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T00:43:35,651 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 6544d6803c2ff354880c0eae0e049fdc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71064242, jitterRate=0.058939725160598755}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:43:35,651 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:35,651 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 6544d6803c2ff354880c0eae0e049fdc: Running coprocessor pre-open hook at 1733359415637Writing region info on filesystem at 1733359415637Initializing all the Stores at 1733359415637Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359415637Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359415638 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359415638Cleaning up temporary data from old regions at 1733359415647 (+9 ms)Running coprocessor post-open hooks at 1733359415651 (+4 ms)Region opened successfully at 1733359415651 2024-12-05T00:43:35,652 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., pid=142, masterSystemTime=1733359415629 2024-12-05T00:43:35,653 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,653 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:35,654 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=OPEN, openSeqNum=2, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:35,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:43:35,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-05T00:43:35,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 in 179 msec 2024-12-05T00:43:35,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-05T00:43:35,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, ASSIGN in 339 msec 2024-12-05T00:43:35,658 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T00:43:35,658 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359415658"}]},"ts":"1733359415658"} 2024-12-05T00:43:35,659 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T00:43:35,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T00:43:35,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-05T00:43:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T00:43:36,634 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:36,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2d690e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:36,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:36,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:36,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2612b6e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:36,642 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734c5abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:36,644 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:36,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:36,647 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T00:43:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fea72ea5c4b6,41705,1733359266486 2024-12-05T00:43:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b41c77d 2024-12-05T00:43:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T00:43:36,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T00:43:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T00:43:36,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T00:43:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T00:43:36,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742316_1492 (size=1097) 2024-12-05T00:43:37,120 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T00:43:37,121 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T00:43:37,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:43:37,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, REOPEN/MOVE}] 2024-12-05T00:43:37,128 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, REOPEN/MOVE 2024-12-05T00:43:37,129 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, REOPEN/MOVE because future has completed 2024-12-05T00:43:37,131 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:43:37,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:43:37,284 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,285 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:43:37,285 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 6544d6803c2ff354880c0eae0e049fdc, disabling compactions & flushes 2024-12-05T00:43:37,285 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,285 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,285 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. after waiting 0 ms 2024-12-05T00:43:37,285 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,293 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T00:43:37,294 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,294 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 6544d6803c2ff354880c0eae0e049fdc: Waiting for close lock at 1733359417285Running coprocessor pre-close hooks at 1733359417285Disabling compacts and flushes for region at 1733359417285Disabling writes for close at 1733359417285Writing region close event to WAL at 1733359417286 (+1 ms)Running coprocessor post-close hooks at 1733359417294 (+8 ms)Closed at 1733359417294 2024-12-05T00:43:37,294 WARN [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 6544d6803c2ff354880c0eae0e049fdc to self. 2024-12-05T00:43:37,296 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,297 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=CLOSED 2024-12-05T00:43:37,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:43:37,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-05T00:43:37,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 in 169 msec 2024-12-05T00:43:37,303 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, REOPEN/MOVE; state=CLOSED, location=fea72ea5c4b6,42683,1733359267474; forceNewPlan=false, retain=true 2024-12-05T00:43:37,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=OPENING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, REOPEN/MOVE because future has completed 2024-12-05T00:43:37,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:43:37,611 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,611 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} 2024-12-05T00:43:37,611 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,611 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T00:43:37,612 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,612 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,614 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,615 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:37,616 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6544d6803c2ff354880c0eae0e049fdc columnFamilyName A 2024-12-05T00:43:37,618 DEBUG [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:37,619 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(327): Store=6544d6803c2ff354880c0eae0e049fdc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:37,619 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,620 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:37,620 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6544d6803c2ff354880c0eae0e049fdc columnFamilyName B 2024-12-05T00:43:37,621 DEBUG [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:37,621 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(327): Store=6544d6803c2ff354880c0eae0e049fdc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:37,621 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,622 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T00:43:37,622 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6544d6803c2ff354880c0eae0e049fdc columnFamilyName C 2024-12-05T00:43:37,622 DEBUG [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:37,623 INFO [StoreOpener-6544d6803c2ff354880c0eae0e049fdc-1 {}] regionserver.HStore(327): Store=6544d6803c2ff354880c0eae0e049fdc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T00:43:37,623 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,623 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,624 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,625 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,627 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,627 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,627 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T00:43:37,629 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,630 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 6544d6803c2ff354880c0eae0e049fdc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75003270, jitterRate=0.1176358163356781}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T00:43:37,630 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,631 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 6544d6803c2ff354880c0eae0e049fdc: Running coprocessor pre-open hook at 1733359417612Writing region info on filesystem at 1733359417612Initializing all the Stores at 1733359417613 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359417613Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359417614 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733359417614Cleaning up temporary data from old regions at 1733359417627 (+13 ms)Running coprocessor post-open hooks at 1733359417630 (+3 ms)Region opened successfully at 1733359417631 (+1 ms) 2024-12-05T00:43:37,632 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., pid=147, masterSystemTime=1733359417607 2024-12-05T00:43:37,635 DEBUG [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,635 INFO [RS_OPEN_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,635 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=OPEN, openSeqNum=5, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:43:37,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-12-05T00:43:37,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 in 183 msec 2024-12-05T00:43:37,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-05T00:43:37,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, REOPEN/MOVE in 514 msec 2024-12-05T00:43:37,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-05T00:43:37,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-12-05T00:43:37,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 996 msec 2024-12-05T00:43:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-05T00:43:37,650 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:37,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee989f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,651 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52530edf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fb780c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41209cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,653 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78341110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1d217b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a0408c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,656 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4805e80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a8170a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6187a3f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,658 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1820337d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ff44fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6b4274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712f46c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32292c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3d6c0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f2685f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc3b50c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600a9875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ebe710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1709f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222a1478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,669 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b7a535, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,670 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7c4d2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9b51f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,671 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3e9bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336ed2a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac3ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fea72ea5c4b6,41705,-1 for getting cluster id 2024-12-05T00:43:37,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T00:43:37,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '03d73038-d2a3-4039-b0c4-73d5bd1f77b7' 2024-12-05T00:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T00:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "03d73038-d2a3-4039-b0c4-73d5bd1f77b7" 2024-12-05T00:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebd8b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fea72ea5c4b6,41705,-1] 2024-12-05T00:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T00:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:37,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T00:43:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eabb121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T00:43:37,675 DEBUG [Thread-2225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,675 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,675 DEBUG [Thread-2227 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,675 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,675 DEBUG [Thread-2223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,676 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,676 DEBUG [Thread-2227 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,676 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,676 DEBUG [Thread-2227 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,677 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:37,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-05T00:43:37,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,679 DEBUG [Thread-2229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,679 DEBUG [Thread-2231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T00:43:37,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:37,680 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fea72ea5c4b6,42683,1733359267474, seqNum=-1] 2024-12-05T00:43:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T00:43:37,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T00:43:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,683 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:37,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5] 2024-12-05T00:43:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:37,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T00:43:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:37,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359417703, max timestamp=1733359417709Num uniques 859; 2024-12-05T00:43:37,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359417703, max timestamp=1733359417709Num uniques 859; 2024-12-05T00:43:37,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:37,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:37,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733359417703, max timestamp=1733359417709Num uniques 859; 2024-12-05T00:43:37,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T00:43:37,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053c68087da67545309295dd72efcc5e18_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359417689/Put/seqid=0 2024-12-05T00:43:37,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742317_1493 (size=12154) 2024-12-05T00:43:37,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359477745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359477750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359477750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359477751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359477751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:37,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:37,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:37,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:37,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359477861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359477861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359477861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359477861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359477861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:37,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:37,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359478071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359478071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359478071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359478072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359478072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,128 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:38,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053c68087da67545309295dd72efcc5e18_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c68087da67545309295dd72efcc5e18_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:38,131 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/55870226a451477381eddcbab8d38f93, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:38,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/55870226a451477381eddcbab8d38f93 is 175, key is test_row_0/A:col10/1733359417689/Put/seqid=0 2024-12-05T00:43:38,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742318_1494 (size=30955) 2024-12-05T00:43:38,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:38,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359478381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359478381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359478381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359478382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359478382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,535 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/55870226a451477381eddcbab8d38f93 2024-12-05T00:43:38,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/5c4dea42ef7a4b16a172225fe0cca2cb is 50, key is test_row_0/B:col10/1733359417689/Put/seqid=0 2024-12-05T00:43:38,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742319_1495 (size=12001) 2024-12-05T00:43:38,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:38,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359478891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359478891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359478892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359478892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359478892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:38,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:38,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:38,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:38,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:38,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/5c4dea42ef7a4b16a172225fe0cca2cb 2024-12-05T00:43:38,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/a5f52a842221473b9321ed80644b8dd0 is 50, key is test_row_0/C:col10/1733359417689/Put/seqid=0 2024-12-05T00:43:38,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742320_1496 (size=12001) 2024-12-05T00:43:39,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:39,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/a5f52a842221473b9321ed80644b8dd0 2024-12-05T00:43:39,390 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T00:43:39,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/55870226a451477381eddcbab8d38f93 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93 2024-12-05T00:43:39,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93, entries=150, sequenceid=15, filesize=30.2 K 2024-12-05T00:43:39,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/5c4dea42ef7a4b16a172225fe0cca2cb as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/5c4dea42ef7a4b16a172225fe0cca2cb 2024-12-05T00:43:39,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/5c4dea42ef7a4b16a172225fe0cca2cb, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T00:43:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/a5f52a842221473b9321ed80644b8dd0 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a5f52a842221473b9321ed80644b8dd0 2024-12-05T00:43:39,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a5f52a842221473b9321ed80644b8dd0, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T00:43:39,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6544d6803c2ff354880c0eae0e049fdc in 1728ms, sequenceid=15, compaction requested=false 2024-12-05T00:43:39,418 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T00:43:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:39,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T00:43:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:39,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T00:43:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120560d6e16655414cd89b41146e6eda0e76_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:39,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742321_1497 (size=12154) 2024-12-05T00:43:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:39,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359479909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:39,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359479910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359479911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359479911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359479912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:39,915 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:39,915 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:39,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:39,929 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120560d6e16655414cd89b41146e6eda0e76_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120560d6e16655414cd89b41146e6eda0e76_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/624e6d3333a240dd9952241a18d36b28, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/624e6d3333a240dd9952241a18d36b28 is 175, key is test_row_0/A:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:39,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742322_1498 (size=30955) 2024-12-05T00:43:40,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359480021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359480021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359480021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359480231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359480231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359480231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/624e6d3333a240dd9952241a18d36b28 2024-12-05T00:43:40,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/63303102c8f748c2b3c1e9ef62338454 is 50, key is test_row_0/B:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:40,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742323_1499 (size=12001) 2024-12-05T00:43:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359480542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359480542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359480542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:40,744 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/63303102c8f748c2b3c1e9ef62338454 2024-12-05T00:43:40,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/128d0357f55e4f099fc8fb7a826c4215 is 50, key is test_row_0/C:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:40,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742324_1500 (size=12001) 2024-12-05T00:43:40,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/128d0357f55e4f099fc8fb7a826c4215 2024-12-05T00:43:40,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/624e6d3333a240dd9952241a18d36b28 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28 2024-12-05T00:43:40,774 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28, entries=150, sequenceid=51, filesize=30.2 K 2024-12-05T00:43:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/63303102c8f748c2b3c1e9ef62338454 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/63303102c8f748c2b3c1e9ef62338454 2024-12-05T00:43:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,778 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/63303102c8f748c2b3c1e9ef62338454, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T00:43:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/128d0357f55e4f099fc8fb7a826c4215 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/128d0357f55e4f099fc8fb7a826c4215 2024-12-05T00:43:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,783 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/128d0357f55e4f099fc8fb7a826c4215, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T00:43:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,784 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6544d6803c2ff354880c0eae0e049fdc in 1267ms, sequenceid=51, compaction requested=false 2024-12-05T00:43:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-05T00:43:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-05T00:43:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-05T00:43:40,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1030 sec 2024-12-05T00:43:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.1100 sec 2024-12-05T00:43:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:41,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T00:43:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c9a4c4bad3b84dd38e3b319c60aa182b_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_1/A:col10/1733359419907/Put/seqid=0 2024-12-05T00:43:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742325_1501 (size=12154) 2024-12-05T00:43:41,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:41,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c9a4c4bad3b84dd38e3b319c60aa182b_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c9a4c4bad3b84dd38e3b319c60aa182b_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:41,096 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f608bfb4e8f24992be3433ff38ca640f, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:41,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f608bfb4e8f24992be3433ff38ca640f is 175, key is test_row_1/A:col10/1733359419907/Put/seqid=0 2024-12-05T00:43:41,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733359421064, max timestamp=1733359421095Num uniques 859; 2024-12-05T00:43:41,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T00:43:41,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359421088, max timestamp=1733359421095Num uniques 859; 2024-12-05T00:43:41,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733359421088, max timestamp=1733359421095Num uniques 859; 2024-12-05T00:43:41,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:43:41,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fea72ea5c4b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T00:43:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742326_1502 (size=30951) 2024-12-05T00:43:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359481120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359481125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359481126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359481232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359481232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359481232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359481441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359481442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359481442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,503 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f608bfb4e8f24992be3433ff38ca640f 2024-12-05T00:43:41,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/41c76aad318a4209b023e7b620f42051 is 50, key is test_row_1/B:col10/1733359419907/Put/seqid=0 2024-12-05T00:43:41,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742327_1503 (size=9657) 2024-12-05T00:43:41,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359481751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359481751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359481751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T00:43:41,852 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:41,852 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-05T00:43:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T00:43:41,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:41,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:41,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:41,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/41c76aad318a4209b023e7b620f42051 2024-12-05T00:43:41,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/2db5d1e6d5b5412b81ccf700e691f57c is 50, key is test_row_1/C:col10/1733359419907/Put/seqid=0 2024-12-05T00:43:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742328_1504 (size=9657) 2024-12-05T00:43:41,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359481931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359481931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:41,933 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:41,933 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T00:43:42,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T00:43:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T00:43:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T00:43:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359482262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359482262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359482262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:42,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T00:43:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:42,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/2db5d1e6d5b5412b81ccf700e691f57c 2024-12-05T00:43:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f608bfb4e8f24992be3433ff38ca640f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f 2024-12-05T00:43:42,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f, entries=150, sequenceid=62, filesize=30.2 K 2024-12-05T00:43:42,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/41c76aad318a4209b023e7b620f42051 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/41c76aad318a4209b023e7b620f42051 2024-12-05T00:43:42,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/41c76aad318a4209b023e7b620f42051, entries=100, sequenceid=62, filesize=9.4 K 2024-12-05T00:43:42,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/2db5d1e6d5b5412b81ccf700e691f57c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2db5d1e6d5b5412b81ccf700e691f57c 2024-12-05T00:43:42,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2db5d1e6d5b5412b81ccf700e691f57c, entries=100, sequenceid=62, filesize=9.4 K 2024-12-05T00:43:42,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 6544d6803c2ff354880c0eae0e049fdc in 1276ms, sequenceid=62, compaction requested=true 2024-12-05T00:43:42,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:42,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:42,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:42,332 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:42,332 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:42,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:42,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:42,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:42,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:42,333 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:42,333 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/B is initiating minor compaction (all files) 2024-12-05T00:43:42,333 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/B in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,334 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/5c4dea42ef7a4b16a172225fe0cca2cb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/63303102c8f748c2b3c1e9ef62338454, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/41c76aad318a4209b023e7b620f42051] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=32.9 K 2024-12-05T00:43:42,334 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:42,334 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/A is initiating minor compaction (all files) 2024-12-05T00:43:42,334 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/A in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,334 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=90.7 K 2024-12-05T00:43:42,334 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,334 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f] 2024-12-05T00:43:42,334 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c4dea42ef7a4b16a172225fe0cca2cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359417688 2024-12-05T00:43:42,334 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55870226a451477381eddcbab8d38f93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359417688 2024-12-05T00:43:42,335 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 63303102c8f748c2b3c1e9ef62338454, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733359417743 2024-12-05T00:43:42,335 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624e6d3333a240dd9952241a18d36b28, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733359417743 2024-12-05T00:43:42,335 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c76aad318a4209b023e7b620f42051, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359419907 2024-12-05T00:43:42,335 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f608bfb4e8f24992be3433ff38ca640f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359419907 2024-12-05T00:43:42,340 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:42,341 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#B#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:42,342 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/019fb968ff394ed48ce87afcf0dc216b is 50, key is test_row_0/B:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:42,353 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053551a6fb271a47f09b3aaa569cc20ece_6544d6803c2ff354880c0eae0e049fdc store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:42,354 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053551a6fb271a47f09b3aaa569cc20ece_6544d6803c2ff354880c0eae0e049fdc, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:42,354 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053551a6fb271a47f09b3aaa569cc20ece_6544d6803c2ff354880c0eae0e049fdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:42,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742329_1505 (size=12104) 2024-12-05T00:43:42,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742330_1506 (size=4469) 2024-12-05T00:43:42,360 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#A#compaction#432 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:42,360 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/23e0fd3c7cbd4fcc87dc736b675d1515 is 175, key is test_row_0/A:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:42,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742331_1507 (size=31165) 2024-12-05T00:43:42,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,464 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T00:43:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205493d93bccbaf42c99f8706b0863c132c_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359421124/Put/seqid=0 2024-12-05T00:43:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742332_1508 (size=12154) 2024-12-05T00:43:42,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:42,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205493d93bccbaf42c99f8706b0863c132c_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205493d93bccbaf42c99f8706b0863c132c_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/bee28fb8465d4fdc93884b0f9b83cb4b, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:42,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/bee28fb8465d4fdc93884b0f9b83cb4b is 175, key is test_row_0/A:col10/1733359421124/Put/seqid=0 2024-12-05T00:43:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T00:43:42,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742333_1509 (size=30955) 2024-12-05T00:43:42,760 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/019fb968ff394ed48ce87afcf0dc216b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/019fb968ff394ed48ce87afcf0dc216b 2024-12-05T00:43:42,763 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/B of 6544d6803c2ff354880c0eae0e049fdc into 019fb968ff394ed48ce87afcf0dc216b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:42,763 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:42,763 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/B, priority=13, startTime=1733359422332; duration=0sec 2024-12-05T00:43:42,763 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:42,763 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:B 2024-12-05T00:43:42,763 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:42,763 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:42,763 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/C is initiating minor compaction (all files) 2024-12-05T00:43:42,764 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/C in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:42,764 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a5f52a842221473b9321ed80644b8dd0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/128d0357f55e4f099fc8fb7a826c4215, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2db5d1e6d5b5412b81ccf700e691f57c] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=32.9 K 2024-12-05T00:43:42,764 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f52a842221473b9321ed80644b8dd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733359417688 2024-12-05T00:43:42,764 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 128d0357f55e4f099fc8fb7a826c4215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733359417743 2024-12-05T00:43:42,764 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db5d1e6d5b5412b81ccf700e691f57c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359419907 2024-12-05T00:43:42,767 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/23e0fd3c7cbd4fcc87dc736b675d1515 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/23e0fd3c7cbd4fcc87dc736b675d1515 2024-12-05T00:43:42,769 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#C#compaction#434 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:42,770 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/0608aa2629f549eea7d17f0fe380415e is 50, key is test_row_0/C:col10/1733359417750/Put/seqid=0 2024-12-05T00:43:42,771 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/A of 6544d6803c2ff354880c0eae0e049fdc into 23e0fd3c7cbd4fcc87dc736b675d1515(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:42,771 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:42,771 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/A, priority=13, startTime=1733359422332; duration=0sec 2024-12-05T00:43:42,771 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:42,771 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:A 2024-12-05T00:43:42,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742334_1510 (size=12104) 2024-12-05T00:43:42,783 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/0608aa2629f549eea7d17f0fe380415e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/0608aa2629f549eea7d17f0fe380415e 2024-12-05T00:43:42,786 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/C of 6544d6803c2ff354880c0eae0e049fdc into 0608aa2629f549eea7d17f0fe380415e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:42,786 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:42,786 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/C, priority=13, startTime=1733359422333; duration=0sec 2024-12-05T00:43:42,786 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:42,786 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:C 2024-12-05T00:43:42,885 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/bee28fb8465d4fdc93884b0f9b83cb4b 2024-12-05T00:43:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/3295e927d746431f8130e5857ce02e97 is 50, key is test_row_0/B:col10/1733359421124/Put/seqid=0 2024-12-05T00:43:42,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742335_1511 (size=12001) 2024-12-05T00:43:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T00:43:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359483282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359483283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359483284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/3295e927d746431f8130e5857ce02e97 2024-12-05T00:43:43,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/f360daf4b1424b32bbe81459f38c6df6 is 50, key is test_row_0/C:col10/1733359421124/Put/seqid=0 2024-12-05T00:43:43,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742336_1512 (size=12001) 2024-12-05T00:43:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359483392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359483392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359483392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359483602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359483602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359483602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/f360daf4b1424b32bbe81459f38c6df6 2024-12-05T00:43:43,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/bee28fb8465d4fdc93884b0f9b83cb4b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b 2024-12-05T00:43:43,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b, entries=150, sequenceid=97, filesize=30.2 K 2024-12-05T00:43:43,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/3295e927d746431f8130e5857ce02e97 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/3295e927d746431f8130e5857ce02e97 2024-12-05T00:43:43,719 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/3295e927d746431f8130e5857ce02e97, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:43:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/f360daf4b1424b32bbe81459f38c6df6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/f360daf4b1424b32bbe81459f38c6df6 2024-12-05T00:43:43,723 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/f360daf4b1424b32bbe81459f38c6df6, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T00:43:43,723 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 6544d6803c2ff354880c0eae0e049fdc in 1259ms, sequenceid=97, compaction requested=false 2024-12-05T00:43:43,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:43,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:43,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-05T00:43:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-05T00:43:43,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-05T00:43:43,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-12-05T00:43:43,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8750 sec 2024-12-05T00:43:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:43,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T00:43:43,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:43,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:43,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:43,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:43,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:43,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:43,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f8e9ddfd2677459b980788b025ee063e_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:43,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742337_1513 (size=12154) 2024-12-05T00:43:43,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359483932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359483935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359483936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T00:43:44,001 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:44,002 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-05T00:43:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:44,003 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:44,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:44,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:44,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359484042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359484052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359484052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:44,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359484251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359484262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359484262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:44,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:44,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f8e9ddfd2677459b980788b025ee063e_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f8e9ddfd2677459b980788b025ee063e_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:44,332 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/488b68b4de69479bb02b786b71baf812, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:44,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/488b68b4de69479bb02b786b71baf812 is 175, key is test_row_0/A:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:44,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742338_1514 (size=30955) 2024-12-05T00:43:44,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/488b68b4de69479bb02b786b71baf812 2024-12-05T00:43:44,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/54d2f9b4129c4139a8f3327c7da3d1e9 is 50, key is test_row_0/B:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742339_1515 (size=12001) 2024-12-05T00:43:44,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359484562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359484571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359484571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:44,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:44,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/54d2f9b4129c4139a8f3327c7da3d1e9 2024-12-05T00:43:44,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/53f28fe64a4b443ea91b3079462854c4 is 50, key is test_row_0/C:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:44,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742340_1516 (size=12001) 2024-12-05T00:43:44,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:44,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:45,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:45,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:45,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:45,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359485072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359485081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359485081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:45,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/53f28fe64a4b443ea91b3079462854c4 2024-12-05T00:43:45,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/488b68b4de69479bb02b786b71baf812 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812 2024-12-05T00:43:45,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812, entries=150, sequenceid=113, filesize=30.2 K 2024-12-05T00:43:45,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/54d2f9b4129c4139a8f3327c7da3d1e9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/54d2f9b4129c4139a8f3327c7da3d1e9 2024-12-05T00:43:45,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/54d2f9b4129c4139a8f3327c7da3d1e9, entries=150, sequenceid=113, filesize=11.7 K 2024-12-05T00:43:45,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/53f28fe64a4b443ea91b3079462854c4 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/53f28fe64a4b443ea91b3079462854c4 2024-12-05T00:43:45,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/53f28fe64a4b443ea91b3079462854c4, entries=150, sequenceid=113, filesize=11.7 K 2024-12-05T00:43:45,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6544d6803c2ff354880c0eae0e049fdc in 1255ms, sequenceid=113, compaction requested=true 2024-12-05T00:43:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:45,168 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:45,169 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:45,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:45,169 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:45,169 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/B is initiating minor compaction (all files) 2024-12-05T00:43:45,169 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/B in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,169 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/019fb968ff394ed48ce87afcf0dc216b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/3295e927d746431f8130e5857ce02e97, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/54d2f9b4129c4139a8f3327c7da3d1e9] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=35.3 K 2024-12-05T00:43:45,169 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:45,169 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/A is initiating minor compaction (all files) 2024-12-05T00:43:45,170 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/A in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,170 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/23e0fd3c7cbd4fcc87dc736b675d1515, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=90.9 K 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 019fb968ff394ed48ce87afcf0dc216b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359417750 2024-12-05T00:43:45,170 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/23e0fd3c7cbd4fcc87dc736b675d1515, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812] 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 3295e927d746431f8130e5857ce02e97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359421117 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e0fd3c7cbd4fcc87dc736b675d1515, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359417750 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d2f9b4129c4139a8f3327c7da3d1e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733359423282 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting bee28fb8465d4fdc93884b0f9b83cb4b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359421117 2024-12-05T00:43:45,170 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488b68b4de69479bb02b786b71baf812, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733359423282 2024-12-05T00:43:45,174 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:45,175 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#B#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:45,176 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/4b68218aa3674adf8b5850e8ea7a72e5 is 50, key is test_row_0/B:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:45,178 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205927d921950e243c2b0b925b3ddc2290d_6544d6803c2ff354880c0eae0e049fdc store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:45,179 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205927d921950e243c2b0b925b3ddc2290d_6544d6803c2ff354880c0eae0e049fdc, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:45,179 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205927d921950e243c2b0b925b3ddc2290d_6544d6803c2ff354880c0eae0e049fdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:45,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742341_1517 (size=12207) 2024-12-05T00:43:45,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742342_1518 (size=4469) 2024-12-05T00:43:45,185 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/4b68218aa3674adf8b5850e8ea7a72e5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4b68218aa3674adf8b5850e8ea7a72e5 2024-12-05T00:43:45,189 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/B of 6544d6803c2ff354880c0eae0e049fdc into 4b68218aa3674adf8b5850e8ea7a72e5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:45,189 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:45,189 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/B, priority=13, startTime=1733359425168; duration=0sec 2024-12-05T00:43:45,189 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:45,189 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:B 2024-12-05T00:43:45,189 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:45,190 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:45,190 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/C is initiating minor compaction (all files) 2024-12-05T00:43:45,190 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/C in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,190 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/0608aa2629f549eea7d17f0fe380415e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/f360daf4b1424b32bbe81459f38c6df6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/53f28fe64a4b443ea91b3079462854c4] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=35.3 K 2024-12-05T00:43:45,190 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 0608aa2629f549eea7d17f0fe380415e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733359417750 2024-12-05T00:43:45,190 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting f360daf4b1424b32bbe81459f38c6df6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733359421117 2024-12-05T00:43:45,191 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 53f28fe64a4b443ea91b3079462854c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733359423282 2024-12-05T00:43:45,200 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#C#compaction#442 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:45,201 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5 is 50, key is test_row_0/C:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:45,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742343_1519 (size=12207) 2024-12-05T00:43:45,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:45,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205787e7f83329248f1aac995bc452aa418_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359423927/Put/seqid=0 2024-12-05T00:43:45,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742344_1520 (size=12254) 2024-12-05T00:43:45,584 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#A#compaction#441 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:45,585 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/e70e73a3922d445ea187a02553f64493 is 175, key is test_row_0/A:col10/1733359423282/Put/seqid=0 2024-12-05T00:43:45,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742345_1521 (size=31161) 2024-12-05T00:43:45,621 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5 2024-12-05T00:43:45,624 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/C of 6544d6803c2ff354880c0eae0e049fdc into 2e8bdaa16f6d4c7ba2b4ab3a6831d4b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:45,624 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:45,625 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/C, priority=13, startTime=1733359425168; duration=0sec 2024-12-05T00:43:45,625 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:45,625 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:C 2024-12-05T00:43:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:45,634 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205787e7f83329248f1aac995bc452aa418_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205787e7f83329248f1aac995bc452aa418_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/2447d0170d884f009c2c9f4b68de1b7c, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:45,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/2447d0170d884f009c2c9f4b68de1b7c is 175, key is test_row_0/A:col10/1733359423927/Put/seqid=0 2024-12-05T00:43:45,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742346_1522 (size=31055) 2024-12-05T00:43:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:45,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359485974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359485974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:45,991 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/e70e73a3922d445ea187a02553f64493 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/e70e73a3922d445ea187a02553f64493 2024-12-05T00:43:45,994 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/A of 6544d6803c2ff354880c0eae0e049fdc into e70e73a3922d445ea187a02553f64493(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:45,994 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:45,994 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/A, priority=13, startTime=1733359425168; duration=0sec 2024-12-05T00:43:45,994 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:45,994 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:A 2024-12-05T00:43:46,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/2447d0170d884f009c2c9f4b68de1b7c 2024-12-05T00:43:46,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/53115acdd7c047668573c886dca2499a is 50, key is test_row_0/B:col10/1733359423927/Put/seqid=0 2024-12-05T00:43:46,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742347_1523 (size=12101) 2024-12-05T00:43:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359486082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359486082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359486082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,084 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_0' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359486091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,093 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359486101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,103 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_2' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:46,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359486292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359486292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,447 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/53115acdd7c047668573c886dca2499a 2024-12-05T00:43:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/c232737ee70f4d0b8a2364b197f7c762 is 50, key is test_row_0/C:col10/1733359423927/Put/seqid=0 2024-12-05T00:43:46,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742348_1524 (size=12101) 2024-12-05T00:43:46,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/c232737ee70f4d0b8a2364b197f7c762 2024-12-05T00:43:46,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/2447d0170d884f009c2c9f4b68de1b7c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c 2024-12-05T00:43:46,463 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c, entries=150, sequenceid=133, filesize=30.3 K 2024-12-05T00:43:46,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/53115acdd7c047668573c886dca2499a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/53115acdd7c047668573c886dca2499a 2024-12-05T00:43:46,466 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/53115acdd7c047668573c886dca2499a, entries=150, sequenceid=133, filesize=11.8 K 2024-12-05T00:43:46,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/c232737ee70f4d0b8a2364b197f7c762 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c232737ee70f4d0b8a2364b197f7c762 2024-12-05T00:43:46,470 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c232737ee70f4d0b8a2364b197f7c762, entries=150, sequenceid=133, filesize=11.8 K 2024-12-05T00:43:46,471 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6544d6803c2ff354880c0eae0e049fdc in 1247ms, sequenceid=133, compaction requested=false 2024-12-05T00:43:46,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:46,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:46,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-05T00:43:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-05T00:43:46,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-05T00:43:46,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-12-05T00:43:46,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4720 sec 2024-12-05T00:43:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:46,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T00:43:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:46,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a97595fe8f5b4510847ebec157c8f9ac_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:46,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742349_1525 (size=14794) 2024-12-05T00:43:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359486626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359486629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359486731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359486742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359486941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:46,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359486951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:47,013 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:47,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a97595fe8f5b4510847ebec157c8f9ac_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a97595fe8f5b4510847ebec157c8f9ac_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:47,089 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f132e68bae2d4bbcad69e0c68db928b6, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:47,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f132e68bae2d4bbcad69e0c68db928b6 is 175, key is test_row_0/A:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:47,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742350_1526 (size=39749) 2024-12-05T00:43:47,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359487252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:47,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359487262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:47,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f132e68bae2d4bbcad69e0c68db928b6 2024-12-05T00:43:47,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/39636caefbaa44b987944f388baadaea is 50, key is test_row_0/B:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:47,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742351_1527 (size=12151) 2024-12-05T00:43:47,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359487761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:47,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359487772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:47,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/39636caefbaa44b987944f388baadaea 2024-12-05T00:43:47,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/71e5d8d152e643fa817a9cd9de165535 is 50, key is test_row_0/C:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:47,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742352_1528 (size=12151) 2024-12-05T00:43:47,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/71e5d8d152e643fa817a9cd9de165535 2024-12-05T00:43:47,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f132e68bae2d4bbcad69e0c68db928b6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6 2024-12-05T00:43:47,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6, entries=200, sequenceid=150, filesize=38.8 K 2024-12-05T00:43:47,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/39636caefbaa44b987944f388baadaea as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/39636caefbaa44b987944f388baadaea 2024-12-05T00:43:47,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/39636caefbaa44b987944f388baadaea, entries=150, sequenceid=150, filesize=11.9 K 2024-12-05T00:43:47,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/71e5d8d152e643fa817a9cd9de165535 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/71e5d8d152e643fa817a9cd9de165535 2024-12-05T00:43:47,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/71e5d8d152e643fa817a9cd9de165535, entries=150, sequenceid=150, filesize=11.9 K 2024-12-05T00:43:47,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6544d6803c2ff354880c0eae0e049fdc in 1323ms, sequenceid=150, compaction requested=true 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:47,926 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:47,926 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:47,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:47,927 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:47,927 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/B is initiating minor compaction (all files) 2024-12-05T00:43:47,927 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/B in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:47,927 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4b68218aa3674adf8b5850e8ea7a72e5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/53115acdd7c047668573c886dca2499a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/39636caefbaa44b987944f388baadaea] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=35.6 K 2024-12-05T00:43:47,927 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:47,928 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/A is initiating minor compaction (all files) 2024-12-05T00:43:47,928 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/A in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:47,928 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/e70e73a3922d445ea187a02553f64493, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=99.6 K 2024-12-05T00:43:47,928 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:47,928 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/e70e73a3922d445ea187a02553f64493, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6] 2024-12-05T00:43:47,928 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b68218aa3674adf8b5850e8ea7a72e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733359423282 2024-12-05T00:43:47,928 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting e70e73a3922d445ea187a02553f64493, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733359423282 2024-12-05T00:43:47,928 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 53115acdd7c047668573c886dca2499a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733359423927 2024-12-05T00:43:47,928 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2447d0170d884f009c2c9f4b68de1b7c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733359423927 2024-12-05T00:43:47,929 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 39636caefbaa44b987944f388baadaea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733359425964 2024-12-05T00:43:47,929 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f132e68bae2d4bbcad69e0c68db928b6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733359425964 2024-12-05T00:43:47,934 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:47,935 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:47,935 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/2664e9c2f09d4ab5ad7bf2aad7444e1a is 50, key is test_row_0/B:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:47,940 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120542ad5fdc177c4feda4fd266668f88804_6544d6803c2ff354880c0eae0e049fdc store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:47,941 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120542ad5fdc177c4feda4fd266668f88804_6544d6803c2ff354880c0eae0e049fdc, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:47,941 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120542ad5fdc177c4feda4fd266668f88804_6544d6803c2ff354880c0eae0e049fdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742353_1529 (size=12459) 2024-12-05T00:43:47,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742354_1530 (size=4469) 2024-12-05T00:43:47,956 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#A#compaction#450 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:47,957 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/1944e29f262c44fdb5d6785105bebe33 is 175, key is test_row_0/A:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742355_1531 (size=31413) 2024-12-05T00:43:47,963 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/1944e29f262c44fdb5d6785105bebe33 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/1944e29f262c44fdb5d6785105bebe33 2024-12-05T00:43:47,967 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/A of 6544d6803c2ff354880c0eae0e049fdc into 1944e29f262c44fdb5d6785105bebe33(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:47,967 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:47,967 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/A, priority=13, startTime=1733359427926; duration=0sec 2024-12-05T00:43:47,967 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:47,967 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:A 2024-12-05T00:43:47,967 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T00:43:47,968 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T00:43:47,968 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/C is initiating minor compaction (all files) 2024-12-05T00:43:47,968 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/C in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:47,968 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c232737ee70f4d0b8a2364b197f7c762, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/71e5d8d152e643fa817a9cd9de165535] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=35.6 K 2024-12-05T00:43:47,969 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e8bdaa16f6d4c7ba2b4ab3a6831d4b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733359423282 2024-12-05T00:43:47,969 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting c232737ee70f4d0b8a2364b197f7c762, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733359423927 2024-12-05T00:43:47,969 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e5d8d152e643fa817a9cd9de165535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733359425964 2024-12-05T00:43:47,975 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:47,976 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/edbcac339d6d482c964452ea581c6e34 is 50, key is test_row_0/C:col10/1733359426603/Put/seqid=0 2024-12-05T00:43:47,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742356_1532 (size=12459) 2024-12-05T00:43:47,982 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/edbcac339d6d482c964452ea581c6e34 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/edbcac339d6d482c964452ea581c6e34 2024-12-05T00:43:47,986 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/C of 6544d6803c2ff354880c0eae0e049fdc into edbcac339d6d482c964452ea581c6e34(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:47,986 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:47,986 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/C, priority=13, startTime=1733359427926; duration=0sec 2024-12-05T00:43:47,987 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:47,987 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:C 2024-12-05T00:43:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:48,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T00:43:48,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:48,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:48,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:48,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d3dbe6255f5b41feb156a57458e133b9_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359428102/Put/seqid=0 2024-12-05T00:43:48,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742357_1533 (size=14794) 2024-12-05T00:43:48,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:48,115 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d3dbe6255f5b41feb156a57458e133b9_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d3dbe6255f5b41feb156a57458e133b9_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:48,116 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f4785ca3144f499e82539aab70ecfca6, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:48,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f4785ca3144f499e82539aab70ecfca6 is 175, key is test_row_0/A:col10/1733359428102/Put/seqid=0 2024-12-05T00:43:48,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742358_1534 (size=39749) 2024-12-05T00:43:48,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359488125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359488126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359488126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T00:43:48,172 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:48,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-05T00:43:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:48,174 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:48,174 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:48,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:48,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359488231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359488232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359488232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:48,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T00:43:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,352 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/2664e9c2f09d4ab5ad7bf2aad7444e1a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/2664e9c2f09d4ab5ad7bf2aad7444e1a 2024-12-05T00:43:48,355 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/B of 6544d6803c2ff354880c0eae0e049fdc into 2664e9c2f09d4ab5ad7bf2aad7444e1a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:48,355 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:48,355 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/B, priority=13, startTime=1733359427926; duration=0sec 2024-12-05T00:43:48,355 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:48,355 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:B 2024-12-05T00:43:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359488441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359488441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359488442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T00:43:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:48,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f4785ca3144f499e82539aab70ecfca6 2024-12-05T00:43:48,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/e1a236d4099e45059b413bf09f31ab46 is 50, key is test_row_0/B:col10/1733359428102/Put/seqid=0 2024-12-05T00:43:48,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742359_1535 (size=12151) 2024-12-05T00:43:48,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T00:43:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359488751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359488751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359488751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359488771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,774 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T00:43:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359488792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:48,794 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:48,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/e1a236d4099e45059b413bf09f31ab46 2024-12-05T00:43:48,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/04b376e8e6954528b632095a4b9b3d3d is 50, key is test_row_0/C:col10/1733359428102/Put/seqid=0 2024-12-05T00:43:48,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T00:43:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:48,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:48,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742360_1536 (size=12151) 2024-12-05T00:43:48,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/04b376e8e6954528b632095a4b9b3d3d 2024-12-05T00:43:48,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f4785ca3144f499e82539aab70ecfca6 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6 2024-12-05T00:43:48,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6, entries=200, sequenceid=170, filesize=38.8 K 2024-12-05T00:43:48,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/e1a236d4099e45059b413bf09f31ab46 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/e1a236d4099e45059b413bf09f31ab46 2024-12-05T00:43:48,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/e1a236d4099e45059b413bf09f31ab46, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T00:43:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/04b376e8e6954528b632095a4b9b3d3d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/04b376e8e6954528b632095a4b9b3d3d 2024-12-05T00:43:48,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/04b376e8e6954528b632095a4b9b3d3d, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T00:43:48,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 6544d6803c2ff354880c0eae0e049fdc in 849ms, sequenceid=170, compaction requested=false 2024-12-05T00:43:48,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:49,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:49,089 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1e47ef5d2584344a5ae3ea30b0d3fdc_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359428125/Put/seqid=0 2024-12-05T00:43:49,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742361_1537 (size=12304) 2024-12-05T00:43:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359489279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359489280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359489282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:49,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359489391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359489391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359489391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:49,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1e47ef5d2584344a5ae3ea30b0d3fdc_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1e47ef5d2584344a5ae3ea30b0d3fdc_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:49,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/54b155f4ebfb4f56aa7c116c03af600e, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/54b155f4ebfb4f56aa7c116c03af600e is 175, key is test_row_0/A:col10/1733359428125/Put/seqid=0 2024-12-05T00:43:49,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742362_1538 (size=31105) 2024-12-05T00:43:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359489601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359489601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359489601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,907 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/54b155f4ebfb4f56aa7c116c03af600e 2024-12-05T00:43:49,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/595fc3f66f7849c5b62a1f88ccac675f is 50, key is test_row_0/B:col10/1733359428125/Put/seqid=0 2024-12-05T00:43:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359489911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359489912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359489912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:49,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742363_1539 (size=12151) 2024-12-05T00:43:50,316 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/595fc3f66f7849c5b62a1f88ccac675f 2024-12-05T00:43:50,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:50,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/5c02a9265a0547949741631788848b74 is 50, key is test_row_0/C:col10/1733359428125/Put/seqid=0 2024-12-05T00:43:50,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742364_1540 (size=12151) 2024-12-05T00:43:50,325 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/5c02a9265a0547949741631788848b74 2024-12-05T00:43:50,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/54b155f4ebfb4f56aa7c116c03af600e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e 2024-12-05T00:43:50,331 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e, entries=150, sequenceid=186, filesize=30.4 K 2024-12-05T00:43:50,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/595fc3f66f7849c5b62a1f88ccac675f as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/595fc3f66f7849c5b62a1f88ccac675f 2024-12-05T00:43:50,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/595fc3f66f7849c5b62a1f88ccac675f, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T00:43:50,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/5c02a9265a0547949741631788848b74 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/5c02a9265a0547949741631788848b74 2024-12-05T00:43:50,338 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/5c02a9265a0547949741631788848b74, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T00:43:50,339 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6544d6803c2ff354880c0eae0e049fdc in 1249ms, sequenceid=186, compaction requested=true 2024-12-05T00:43:50,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:50,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:50,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-05T00:43:50,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-05T00:43:50,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-05T00:43:50,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1660 sec 2024-12-05T00:43:50,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1690 sec 2024-12-05T00:43:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:50,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T00:43:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:50,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056d8a4ac0da4642169c2fd9815143edd3_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:50,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742365_1541 (size=14794) 2024-12-05T00:43:50,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:50,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056d8a4ac0da4642169c2fd9815143edd3_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056d8a4ac0da4642169c2fd9815143edd3_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:50,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/8dff1d0caa844571adf0c1f75c0a84d5, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:50,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/8dff1d0caa844571adf0c1f75c0a84d5 is 175, key is test_row_0/A:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:50,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359490435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359490437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742366_1542 (size=39749) 2024-12-05T00:43:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359490439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359490542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359490551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359490551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359490752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359490762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359490762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359490781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,784 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359490822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:50,824 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:50,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/8dff1d0caa844571adf0c1f75c0a84d5 2024-12-05T00:43:50,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/7e868b655f7f4f1b8a2400bffd4c76ef is 50, key is test_row_0/B:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:50,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742367_1543 (size=12151) 2024-12-05T00:43:51,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359491061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359491072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:51,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359491072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:51,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/7e868b655f7f4f1b8a2400bffd4c76ef 2024-12-05T00:43:51,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/30b9402cf5da49cebbc32a94fbf7e76b is 50, key is test_row_0/C:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:51,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742368_1544 (size=12151) 2024-12-05T00:43:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359491571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359491581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359491581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:51,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/30b9402cf5da49cebbc32a94fbf7e76b 2024-12-05T00:43:51,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/8dff1d0caa844571adf0c1f75c0a84d5 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5 2024-12-05T00:43:51,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5, entries=200, sequenceid=205, filesize=38.8 K 2024-12-05T00:43:51,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/7e868b655f7f4f1b8a2400bffd4c76ef as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/7e868b655f7f4f1b8a2400bffd4c76ef 2024-12-05T00:43:51,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/7e868b655f7f4f1b8a2400bffd4c76ef, entries=150, sequenceid=205, filesize=11.9 K 2024-12-05T00:43:51,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/30b9402cf5da49cebbc32a94fbf7e76b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/30b9402cf5da49cebbc32a94fbf7e76b 2024-12-05T00:43:51,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/30b9402cf5da49cebbc32a94fbf7e76b, entries=150, sequenceid=205, filesize=11.9 K 2024-12-05T00:43:51,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6544d6803c2ff354880c0eae0e049fdc in 1248ms, sequenceid=205, compaction requested=true 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:51,672 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:51,672 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:51,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:51,673 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:51,673 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:51,673 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/A is initiating minor compaction (all files) 2024-12-05T00:43:51,673 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/B is initiating minor compaction (all files) 2024-12-05T00:43:51,673 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/B in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:51,673 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/A in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:51,673 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/2664e9c2f09d4ab5ad7bf2aad7444e1a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/e1a236d4099e45059b413bf09f31ab46, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/595fc3f66f7849c5b62a1f88ccac675f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/7e868b655f7f4f1b8a2400bffd4c76ef] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=47.8 K 2024-12-05T00:43:51,673 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/1944e29f262c44fdb5d6785105bebe33, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=138.7 K 2024-12-05T00:43:51,673 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/1944e29f262c44fdb5d6785105bebe33, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5] 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 2664e9c2f09d4ab5ad7bf2aad7444e1a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733359425964 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1944e29f262c44fdb5d6785105bebe33, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733359425964 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a236d4099e45059b413bf09f31ab46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359426615 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4785ca3144f499e82539aab70ecfca6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359426615 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 595fc3f66f7849c5b62a1f88ccac675f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733359428124 2024-12-05T00:43:51,674 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b155f4ebfb4f56aa7c116c03af600e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733359428124 2024-12-05T00:43:51,675 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e868b655f7f4f1b8a2400bffd4c76ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733359429279 2024-12-05T00:43:51,675 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dff1d0caa844571adf0c1f75c0a84d5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733359429279 2024-12-05T00:43:51,685 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:51,687 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205814f2bdb7dba43abb222b840bddea9eb_6544d6803c2ff354880c0eae0e049fdc store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:51,687 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#B#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:51,688 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/20e415fbd78145c9a52b6149b74cfe2c is 50, key is test_row_0/B:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:51,689 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205814f2bdb7dba43abb222b840bddea9eb_6544d6803c2ff354880c0eae0e049fdc, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:51,690 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205814f2bdb7dba43abb222b840bddea9eb_6544d6803c2ff354880c0eae0e049fdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742369_1545 (size=12595) 2024-12-05T00:43:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742370_1546 (size=4469) 2024-12-05T00:43:52,093 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:52,094 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/9a0f4ee438eb4b34a1c907586521759d is 175, key is test_row_0/A:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:52,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742371_1547 (size=31549) 2024-12-05T00:43:52,136 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/20e415fbd78145c9a52b6149b74cfe2c as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/20e415fbd78145c9a52b6149b74cfe2c 2024-12-05T00:43:52,139 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/B of 6544d6803c2ff354880c0eae0e049fdc into 20e415fbd78145c9a52b6149b74cfe2c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:52,139 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:52,139 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/B, priority=12, startTime=1733359431672; duration=0sec 2024-12-05T00:43:52,139 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:52,139 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:B 2024-12-05T00:43:52,139 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:52,140 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:52,140 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/C is initiating minor compaction (all files) 2024-12-05T00:43:52,140 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/C in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:52,140 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/edbcac339d6d482c964452ea581c6e34, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/04b376e8e6954528b632095a4b9b3d3d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/5c02a9265a0547949741631788848b74, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/30b9402cf5da49cebbc32a94fbf7e76b] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=47.8 K 2024-12-05T00:43:52,140 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting edbcac339d6d482c964452ea581c6e34, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733359425964 2024-12-05T00:43:52,140 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b376e8e6954528b632095a4b9b3d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733359426615 2024-12-05T00:43:52,141 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c02a9265a0547949741631788848b74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733359428124 2024-12-05T00:43:52,141 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b9402cf5da49cebbc32a94fbf7e76b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733359429279 2024-12-05T00:43:52,146 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:52,146 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/c6598abb5ffd4d97b1b049c28536f2c1 is 50, key is test_row_0/C:col10/1733359429279/Put/seqid=0 2024-12-05T00:43:52,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742372_1548 (size=12595) 2024-12-05T00:43:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T00:43:52,332 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:52,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-05T00:43:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T00:43:52,333 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:52,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:52,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T00:43:52,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T00:43:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:52,486 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T00:43:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:52,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120528b167b67bfc4a388d31567ce91a52cc_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359430430/Put/seqid=0 2024-12-05T00:43:52,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742373_1549 (size=12304) 2024-12-05T00:43:52,540 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/9a0f4ee438eb4b34a1c907586521759d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a0f4ee438eb4b34a1c907586521759d 2024-12-05T00:43:52,543 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/A of 6544d6803c2ff354880c0eae0e049fdc into 9a0f4ee438eb4b34a1c907586521759d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:52,543 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:52,543 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/A, priority=12, startTime=1733359431672; duration=0sec 2024-12-05T00:43:52,544 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:52,544 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:A 2024-12-05T00:43:52,562 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/c6598abb5ffd4d97b1b049c28536f2c1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c6598abb5ffd4d97b1b049c28536f2c1 2024-12-05T00:43:52,566 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/C of 6544d6803c2ff354880c0eae0e049fdc into c6598abb5ffd4d97b1b049c28536f2c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:52,566 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:52,566 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/C, priority=12, startTime=1733359431672; duration=0sec 2024-12-05T00:43:52,566 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:52,566 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:C 2024-12-05T00:43:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:52,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359492607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359492607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359492608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T00:43:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359492722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359492722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359492722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:52,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120528b167b67bfc4a388d31567ce91a52cc_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120528b167b67bfc4a388d31567ce91a52cc_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:52,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/76622f4ded984a2687ff4e7ff98bd7b8, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:52,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/76622f4ded984a2687ff4e7ff98bd7b8 is 175, key is test_row_0/A:col10/1733359430430/Put/seqid=0 2024-12-05T00:43:52,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742374_1550 (size=31105) 2024-12-05T00:43:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359492932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359492932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359492932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T00:43:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359493241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:53,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359493241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359493241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:53,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/76622f4ded984a2687ff4e7ff98bd7b8 2024-12-05T00:43:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/4c42ce3a0d5942e986f3e5f6434b3c1d is 50, key is test_row_0/B:col10/1733359430430/Put/seqid=0 2024-12-05T00:43:53,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742375_1551 (size=12151) 2024-12-05T00:43:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T00:43:53,720 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/4c42ce3a0d5942e986f3e5f6434b3c1d 2024-12-05T00:43:53,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/b9e9406017af43ec96297b839c8c5f54 is 50, key is test_row_0/C:col10/1733359430430/Put/seqid=0 2024-12-05T00:43:53,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742376_1552 (size=12151) 2024-12-05T00:43:53,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359493751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:53,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359493751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359493751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/b9e9406017af43ec96297b839c8c5f54 2024-12-05T00:43:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/76622f4ded984a2687ff4e7ff98bd7b8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8 2024-12-05T00:43:54,135 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8, entries=150, sequenceid=219, filesize=30.4 K 2024-12-05T00:43:54,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/4c42ce3a0d5942e986f3e5f6434b3c1d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4c42ce3a0d5942e986f3e5f6434b3c1d 2024-12-05T00:43:54,139 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4c42ce3a0d5942e986f3e5f6434b3c1d, entries=150, sequenceid=219, filesize=11.9 K 2024-12-05T00:43:54,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/b9e9406017af43ec96297b839c8c5f54 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/b9e9406017af43ec96297b839c8c5f54 2024-12-05T00:43:54,144 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/b9e9406017af43ec96297b839c8c5f54, entries=150, sequenceid=219, filesize=11.9 K 2024-12-05T00:43:54,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6544d6803c2ff354880c0eae0e049fdc in 1658ms, sequenceid=219, compaction requested=false 2024-12-05T00:43:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-05T00:43:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-05T00:43:54,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-05T00:43:54,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-12-05T00:43:54,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8170 sec 2024-12-05T00:43:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T00:43:54,482 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:54,482 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-05T00:43:54,483 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T00:43:54,484 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:54,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T00:43:54,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T00:43:54,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:54,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T00:43:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054cc5a4d00bf64a78800d3cdffd40eb09_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359432605/Put/seqid=0 2024-12-05T00:43:54,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742377_1553 (size=12304) 2024-12-05T00:43:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:54,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359494778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359494778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359494780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T00:43:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1733359494822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,825 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1733359494842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,843 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fea72ea5c4b6,42683,1733359267474 for 'test_row_1' in 6544d6803c2ff354880c0eae0e049fdc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T00:43:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359494891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359494891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359494892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:55,048 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054cc5a4d00bf64a78800d3cdffd40eb09_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054cc5a4d00bf64a78800d3cdffd40eb09_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:55,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f67beb79d54349d8abc9b85b136a20f2, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:55,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f67beb79d54349d8abc9b85b136a20f2 is 175, key is test_row_0/A:col10/1733359432605/Put/seqid=0 2024-12-05T00:43:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742378_1554 (size=31105) 2024-12-05T00:43:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359495101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359495101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359495101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T00:43:55,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359495412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359495412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359495412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f67beb79d54349d8abc9b85b136a20f2 2024-12-05T00:43:55,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/91fdb434e9d348099f1fb97ffebbcb6b is 50, key is test_row_0/B:col10/1733359432605/Put/seqid=0 2024-12-05T00:43:55,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742379_1555 (size=12151) 2024-12-05T00:43:55,470 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/91fdb434e9d348099f1fb97ffebbcb6b 2024-12-05T00:43:55,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/6b266f6efba342d19505050aa6bca842 is 50, key is test_row_0/C:col10/1733359432605/Put/seqid=0 2024-12-05T00:43:55,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742380_1556 (size=12151) 2024-12-05T00:43:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T00:43:55,882 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/6b266f6efba342d19505050aa6bca842 2024-12-05T00:43:55,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/f67beb79d54349d8abc9b85b136a20f2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2 2024-12-05T00:43:55,888 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2, entries=150, sequenceid=240, filesize=30.4 K 2024-12-05T00:43:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/91fdb434e9d348099f1fb97ffebbcb6b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/91fdb434e9d348099f1fb97ffebbcb6b 2024-12-05T00:43:55,892 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/91fdb434e9d348099f1fb97ffebbcb6b, entries=150, sequenceid=240, filesize=11.9 K 2024-12-05T00:43:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/6b266f6efba342d19505050aa6bca842 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/6b266f6efba342d19505050aa6bca842 2024-12-05T00:43:55,895 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/6b266f6efba342d19505050aa6bca842, entries=150, sequenceid=240, filesize=11.9 K 2024-12-05T00:43:55,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6544d6803c2ff354880c0eae0e049fdc in 1258ms, sequenceid=240, compaction requested=true 2024-12-05T00:43:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-05T00:43:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-05T00:43:55,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-05T00:43:55,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-12-05T00:43:55,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4170 sec 2024-12-05T00:43:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:55,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T00:43:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:55,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051f0506db12f64048bb38ca64c2562728_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742381_1557 (size=17284) 2024-12-05T00:43:55,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:55,935 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051f0506db12f64048bb38ca64c2562728_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051f0506db12f64048bb38ca64c2562728_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:55,936 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/9a2f697ee6484aef8291f6b9356f2287, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:55,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/9a2f697ee6484aef8291f6b9356f2287 is 175, key is test_row_0/A:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:55,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742382_1558 (size=48389) 2024-12-05T00:43:55,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359495943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359495946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:55,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359495947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359496051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359496051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359496061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359496261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359496261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359496271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/9a2f697ee6484aef8291f6b9356f2287 2024-12-05T00:43:56,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/6c0b1346ef1b42e29f0eeb50eacf3d2b is 50, key is test_row_0/B:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:56,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742383_1559 (size=12151) 2024-12-05T00:43:56,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359496571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359496572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359496582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T00:43:56,632 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:43:56,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T00:43:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-05T00:43:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:43:56,633 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T00:43:56,634 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T00:43:56,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T00:43:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:43:56,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/6c0b1346ef1b42e29f0eeb50eacf3d2b 2024-12-05T00:43:56,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/eed5b60a3fc6496c86bb63628b2a4e9e is 50, key is test_row_0/C:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:56,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742384_1560 (size=12151) 2024-12-05T00:43:56,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T00:43:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:56,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:56,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T00:43:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:56,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:43:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53004 deadline: 1733359497082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53062 deadline: 1733359497082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:57,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T00:43:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T00:43:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T00:43:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42683 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53006 deadline: 1733359497091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:43:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 , the old value is region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T00:43:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., hostname=fea72ea5c4b6,42683,1733359267474, seqNum=5 because the exception is null or not the one we care about 2024-12-05T00:43:57,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/eed5b60a3fc6496c86bb63628b2a4e9e 2024-12-05T00:43:57,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/9a2f697ee6484aef8291f6b9356f2287 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287 2024-12-05T00:43:57,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287, entries=250, sequenceid=256, filesize=47.3 K 2024-12-05T00:43:57,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/6c0b1346ef1b42e29f0eeb50eacf3d2b as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/6c0b1346ef1b42e29f0eeb50eacf3d2b 2024-12-05T00:43:57,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/6c0b1346ef1b42e29f0eeb50eacf3d2b, entries=150, sequenceid=256, filesize=11.9 K 2024-12-05T00:43:57,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/eed5b60a3fc6496c86bb63628b2a4e9e as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/eed5b60a3fc6496c86bb63628b2a4e9e 2024-12-05T00:43:57,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/eed5b60a3fc6496c86bb63628b2a4e9e, entries=150, sequenceid=256, filesize=11.9 K 2024-12-05T00:43:57,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 6544d6803c2ff354880c0eae0e049fdc in 1247ms, sequenceid=256, compaction requested=true 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:57,171 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6544d6803c2ff354880c0eae0e049fdc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T00:43:57,171 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:57,172 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:57,172 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/A is initiating minor compaction (all files) 2024-12-05T00:43:57,172 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/A in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,172 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a0f4ee438eb4b34a1c907586521759d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=138.8 K 2024-12-05T00:43:57,172 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,172 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. files: [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a0f4ee438eb4b34a1c907586521759d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287] 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/B is initiating minor compaction (all files) 2024-12-05T00:43:57,173 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/B in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,173 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/20e415fbd78145c9a52b6149b74cfe2c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4c42ce3a0d5942e986f3e5f6434b3c1d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/91fdb434e9d348099f1fb97ffebbcb6b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/6c0b1346ef1b42e29f0eeb50eacf3d2b] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=47.9 K 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0f4ee438eb4b34a1c907586521759d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733359429279 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e415fbd78145c9a52b6149b74cfe2c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733359429279 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76622f4ded984a2687ff4e7ff98bd7b8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733359430430 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c42ce3a0d5942e986f3e5f6434b3c1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733359430430 2024-12-05T00:43:57,173 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67beb79d54349d8abc9b85b136a20f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733359432602 2024-12-05T00:43:57,180 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 91fdb434e9d348099f1fb97ffebbcb6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733359432602 2024-12-05T00:43:57,180 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a2f697ee6484aef8291f6b9356f2287, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733359434767 2024-12-05T00:43:57,180 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0b1346ef1b42e29f0eeb50eacf3d2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733359435922 2024-12-05T00:43:57,185 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:57,187 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:57,187 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/fe64f3cbdf614d03ac1543feb2ef908d is 50, key is test_row_0/B:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:57,189 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205dd22ccadc95c444c98e7a8ad13e94fdf_6544d6803c2ff354880c0eae0e049fdc store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742385_1561 (size=12731) 2024-12-05T00:43:57,192 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205dd22ccadc95c444c98e7a8ad13e94fdf_6544d6803c2ff354880c0eae0e049fdc, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:57,193 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dd22ccadc95c444c98e7a8ad13e94fdf_6544d6803c2ff354880c0eae0e049fdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:57,194 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/fe64f3cbdf614d03ac1543feb2ef908d as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/fe64f3cbdf614d03ac1543feb2ef908d 2024-12-05T00:43:57,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742386_1562 (size=4469) 2024-12-05T00:43:57,198 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/B of 6544d6803c2ff354880c0eae0e049fdc into fe64f3cbdf614d03ac1543feb2ef908d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:57,198 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:57,198 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/B, priority=12, startTime=1733359437171; duration=0sec 2024-12-05T00:43:57,198 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T00:43:57,198 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:B 2024-12-05T00:43:57,199 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T00:43:57,199 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T00:43:57,199 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1541): 6544d6803c2ff354880c0eae0e049fdc/C is initiating minor compaction (all files) 2024-12-05T00:43:57,199 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6544d6803c2ff354880c0eae0e049fdc/C in TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,200 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c6598abb5ffd4d97b1b049c28536f2c1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/b9e9406017af43ec96297b839c8c5f54, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/6b266f6efba342d19505050aa6bca842, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/eed5b60a3fc6496c86bb63628b2a4e9e] into tmpdir=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp, totalSize=47.9 K 2024-12-05T00:43:57,200 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting c6598abb5ffd4d97b1b049c28536f2c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733359429279 2024-12-05T00:43:57,200 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e9406017af43ec96297b839c8c5f54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733359430430 2024-12-05T00:43:57,200 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b266f6efba342d19505050aa6bca842, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733359432602 2024-12-05T00:43:57,201 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] compactions.Compactor(225): Compacting eed5b60a3fc6496c86bb63628b2a4e9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733359435922 2024-12-05T00:43:57,207 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:57,207 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/a3872d52f5174a5aac30ffe575cd9a22 is 50, key is test_row_0/C:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:57,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742387_1563 (size=12731) 2024-12-05T00:43:57,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42683 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-05T00:43:57,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:57,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T00:43:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:43:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:43:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:43:57,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:43:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052693a3dde2ed47ca86b7476827275455_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359435941/Put/seqid=0 2024-12-05T00:43:57,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742388_1564 (size=12454) 2024-12-05T00:43:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:43:57,597 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6544d6803c2ff354880c0eae0e049fdc#A#compaction#473 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T00:43:57,597 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/aad283b4a4ec4577a999fc8f1a7d79e2 is 175, key is test_row_0/A:col10/1733359435923/Put/seqid=0 2024-12-05T00:43:57,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742389_1565 (size=31685) 2024-12-05T00:43:57,603 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/aad283b4a4ec4577a999fc8f1a7d79e2 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/aad283b4a4ec4577a999fc8f1a7d79e2 2024-12-05T00:43:57,606 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/A of 6544d6803c2ff354880c0eae0e049fdc into aad283b4a4ec4577a999fc8f1a7d79e2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:57,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:57,606 INFO [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/A, priority=12, startTime=1733359437171; duration=0sec 2024-12-05T00:43:57,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:57,606 DEBUG [RS:0;fea72ea5c4b6:42683-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:A 2024-12-05T00:43:57,616 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/a3872d52f5174a5aac30ffe575cd9a22 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a3872d52f5174a5aac30ffe575cd9a22 2024-12-05T00:43:57,620 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6544d6803c2ff354880c0eae0e049fdc/C of 6544d6803c2ff354880c0eae0e049fdc into a3872d52f5174a5aac30ffe575cd9a22(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T00:43:57,620 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:57,620 INFO [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., storeName=6544d6803c2ff354880c0eae0e049fdc/C, priority=12, startTime=1733359437171; duration=0sec 2024-12-05T00:43:57,620 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T00:43:57,620 DEBUG [RS:0;fea72ea5c4b6:42683-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6544d6803c2ff354880c0eae0e049fdc:C 2024-12-05T00:43:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:43:57,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052693a3dde2ed47ca86b7476827275455_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052693a3dde2ed47ca86b7476827275455_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/525caec657db4bd284368d38d52ab667, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:43:57,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/525caec657db4bd284368d38d52ab667 is 175, key is test_row_0/A:col10/1733359435941/Put/seqid=0 2024-12-05T00:43:57,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742390_1566 (size=31255) 2024-12-05T00:43:57,678 INFO [Thread-2223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2223. 2024-12-05T00:43:57,678 DEBUG [Thread-2223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:57,678 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,678 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:57,679 INFO [Thread-2231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2231. 2024-12-05T00:43:57,679 DEBUG [Thread-2231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:57,679 DEBUG [Thread-2231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,679 DEBUG [Thread-2231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:57,679 INFO [Thread-2225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2225. 2024-12-05T00:43:57,679 DEBUG [Thread-2225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:57,679 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,679 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:57,679 INFO [Thread-2229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2229. 2024-12-05T00:43:57,680 DEBUG [Thread-2229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:57,680 DEBUG [Thread-2229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,680 DEBUG [Thread-2229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:57,680 INFO [Thread-2227 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2227. 2024-12-05T00:43:57,680 DEBUG [Thread-2227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:57,680 DEBUG [Thread-2227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,680 DEBUG [Thread-2227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:57,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:43:58,062 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/525caec657db4bd284368d38d52ab667 2024-12-05T00:43:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/b062c794b0bf4d5592f3d2972a3416b3 is 50, key is test_row_0/B:col10/1733359435941/Put/seqid=0 2024-12-05T00:43:58,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742391_1567 (size=12301) 2024-12-05T00:43:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42683 {}] regionserver.HRegion(8855): Flush requested on 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:43:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. as already flushing 2024-12-05T00:43:58,094 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-12-05T00:43:58,094 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:58,094 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:58,094 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:58,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:58,106 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-12-05T00:43:58,106 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-12-05T00:43:58,107 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:58,107 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:43:58,107 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:58,107 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:58,107 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:58,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:58,107 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:43:58,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:43:58,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/b062c794b0bf4d5592f3d2972a3416b3 2024-12-05T00:43:58,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/ab9cea76ee184af99bafa8ba13e054f7 is 50, key is test_row_0/C:col10/1733359435941/Put/seqid=0 2024-12-05T00:43:58,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742392_1568 (size=12301) 2024-12-05T00:43:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:43:58,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/ab9cea76ee184af99bafa8ba13e054f7 2024-12-05T00:43:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/525caec657db4bd284368d38d52ab667 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/525caec657db4bd284368d38d52ab667 2024-12-05T00:43:58,908 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/525caec657db4bd284368d38d52ab667, entries=150, sequenceid=274, filesize=30.5 K 2024-12-05T00:43:58,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/b062c794b0bf4d5592f3d2972a3416b3 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/b062c794b0bf4d5592f3d2972a3416b3 2024-12-05T00:43:58,911 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/b062c794b0bf4d5592f3d2972a3416b3, entries=150, sequenceid=274, filesize=12.0 K 2024-12-05T00:43:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/ab9cea76ee184af99bafa8ba13e054f7 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/ab9cea76ee184af99bafa8ba13e054f7 2024-12-05T00:43:58,914 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/ab9cea76ee184af99bafa8ba13e054f7, entries=150, sequenceid=274, filesize=12.0 K 2024-12-05T00:43:58,915 INFO [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 6544d6803c2ff354880c0eae0e049fdc in 1672ms, sequenceid=274, compaction requested=false 2024-12-05T00:43:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 6544d6803c2ff354880c0eae0e049fdc: 2024-12-05T00:43:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:43:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fea72ea5c4b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-05T00:43:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-05T00:43:58,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-05T00:43:58,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2820 sec 2024-12-05T00:43:58,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2850 sec 2024-12-05T00:44:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T00:44:00,793 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:44:04,916 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-12-05T00:44:04,916 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:44:04,916 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:04,917 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:04,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:44:04,946 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-12-05T00:44:04,946 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T00:44:04,947 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:04,947 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:04,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:44:04,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4556 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4506 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4587 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4560 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4513 2024-12-05T00:44:04,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T00:44:04,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:04,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:04,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:04,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:04,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:44:04,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T00:44:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T00:44:04,954 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359444954"}]},"ts":"1733359444954"} 2024-12-05T00:44:04,955 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T00:44:04,956 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T00:44:04,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T00:44:04,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, UNASSIGN}] 2024-12-05T00:44:04,958 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, UNASSIGN 2024-12-05T00:44:04,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=CLOSING, regionLocation=fea72ea5c4b6,42683,1733359267474 2024-12-05T00:44:04,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, UNASSIGN because future has completed 2024-12-05T00:44:04,960 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T00:44:04,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474}] 2024-12-05T00:44:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T00:44:05,113 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:05,113 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T00:44:05,113 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 6544d6803c2ff354880c0eae0e049fdc, disabling compactions & flushes 2024-12-05T00:44:05,113 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:44:05,113 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:44:05,113 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. after waiting 0 ms 2024-12-05T00:44:05,113 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:44:05,114 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 6544d6803c2ff354880c0eae0e049fdc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T00:44:05,114 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=A 2024-12-05T00:44:05,114 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:44:05,114 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=B 2024-12-05T00:44:05,114 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:44:05,114 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6544d6803c2ff354880c0eae0e049fdc, store=C 2024-12-05T00:44:05,115 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T00:44:05,125 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120544a6724cacaa48bf843b7378d90161a6_6544d6803c2ff354880c0eae0e049fdc is 50, key is test_row_0/A:col10/1733359438103/Put/seqid=0 2024-12-05T00:44:05,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742393_1569 (size=12454) 2024-12-05T00:44:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T00:44:05,529 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T00:44:05,537 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120544a6724cacaa48bf843b7378d90161a6_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120544a6724cacaa48bf843b7378d90161a6_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:05,539 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/16af0d3b7eff47b382865512c124b1e1, store: [table=TestAcidGuarantees family=A region=6544d6803c2ff354880c0eae0e049fdc] 2024-12-05T00:44:05,540 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/16af0d3b7eff47b382865512c124b1e1 is 175, key is test_row_0/A:col10/1733359438103/Put/seqid=0 2024-12-05T00:44:05,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742394_1570 (size=31255) 2024-12-05T00:44:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T00:44:05,592 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T00:44:05,947 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/16af0d3b7eff47b382865512c124b1e1 2024-12-05T00:44:05,961 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/107773bebaac4bbc9b7678e5a93153da is 50, key is test_row_0/B:col10/1733359438103/Put/seqid=0 2024-12-05T00:44:05,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742395_1571 (size=12301) 2024-12-05T00:44:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T00:44:06,364 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/107773bebaac4bbc9b7678e5a93153da 2024-12-05T00:44:06,370 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/1ed4d3f3f6ba4011b9482c8591bf133a is 50, key is test_row_0/C:col10/1733359438103/Put/seqid=0 2024-12-05T00:44:06,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742396_1572 (size=12301) 2024-12-05T00:44:06,775 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/1ed4d3f3f6ba4011b9482c8591bf133a 2024-12-05T00:44:06,783 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/A/16af0d3b7eff47b382865512c124b1e1 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/16af0d3b7eff47b382865512c124b1e1 2024-12-05T00:44:06,788 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/16af0d3b7eff47b382865512c124b1e1, entries=150, sequenceid=284, filesize=30.5 K 2024-12-05T00:44:06,788 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/B/107773bebaac4bbc9b7678e5a93153da as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/107773bebaac4bbc9b7678e5a93153da 2024-12-05T00:44:06,792 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/107773bebaac4bbc9b7678e5a93153da, entries=150, sequenceid=284, filesize=12.0 K 2024-12-05T00:44:06,793 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/.tmp/C/1ed4d3f3f6ba4011b9482c8591bf133a as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/1ed4d3f3f6ba4011b9482c8591bf133a 2024-12-05T00:44:06,796 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/1ed4d3f3f6ba4011b9482c8591bf133a, entries=150, sequenceid=284, filesize=12.0 K 2024-12-05T00:44:06,797 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6544d6803c2ff354880c0eae0e049fdc in 1684ms, sequenceid=284, compaction requested=true 2024-12-05T00:44:06,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/23e0fd3c7cbd4fcc87dc736b675d1515, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/e70e73a3922d445ea187a02553f64493, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/1944e29f262c44fdb5d6785105bebe33, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a0f4ee438eb4b34a1c907586521759d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287] to archive 2024-12-05T00:44:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:44:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/55870226a451477381eddcbab8d38f93 2024-12-05T00:44:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/624e6d3333a240dd9952241a18d36b28 2024-12-05T00:44:06,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/23e0fd3c7cbd4fcc87dc736b675d1515 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/23e0fd3c7cbd4fcc87dc736b675d1515 2024-12-05T00:44:06,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f608bfb4e8f24992be3433ff38ca640f 2024-12-05T00:44:06,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/bee28fb8465d4fdc93884b0f9b83cb4b 2024-12-05T00:44:06,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/e70e73a3922d445ea187a02553f64493 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/e70e73a3922d445ea187a02553f64493 2024-12-05T00:44:06,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/488b68b4de69479bb02b786b71baf812 2024-12-05T00:44:06,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/2447d0170d884f009c2c9f4b68de1b7c 2024-12-05T00:44:06,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f132e68bae2d4bbcad69e0c68db928b6 2024-12-05T00:44:06,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/1944e29f262c44fdb5d6785105bebe33 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/1944e29f262c44fdb5d6785105bebe33 2024-12-05T00:44:06,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f4785ca3144f499e82539aab70ecfca6 2024-12-05T00:44:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/54b155f4ebfb4f56aa7c116c03af600e 2024-12-05T00:44:06,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/8dff1d0caa844571adf0c1f75c0a84d5 2024-12-05T00:44:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a0f4ee438eb4b34a1c907586521759d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a0f4ee438eb4b34a1c907586521759d 2024-12-05T00:44:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/76622f4ded984a2687ff4e7ff98bd7b8 2024-12-05T00:44:06,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/f67beb79d54349d8abc9b85b136a20f2 2024-12-05T00:44:06,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/9a2f697ee6484aef8291f6b9356f2287 2024-12-05T00:44:06,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/5c4dea42ef7a4b16a172225fe0cca2cb, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/63303102c8f748c2b3c1e9ef62338454, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/019fb968ff394ed48ce87afcf0dc216b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/41c76aad318a4209b023e7b620f42051, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/3295e927d746431f8130e5857ce02e97, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4b68218aa3674adf8b5850e8ea7a72e5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/54d2f9b4129c4139a8f3327c7da3d1e9, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/53115acdd7c047668573c886dca2499a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/2664e9c2f09d4ab5ad7bf2aad7444e1a, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/39636caefbaa44b987944f388baadaea, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/e1a236d4099e45059b413bf09f31ab46, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/595fc3f66f7849c5b62a1f88ccac675f, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/20e415fbd78145c9a52b6149b74cfe2c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/7e868b655f7f4f1b8a2400bffd4c76ef, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4c42ce3a0d5942e986f3e5f6434b3c1d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/91fdb434e9d348099f1fb97ffebbcb6b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/6c0b1346ef1b42e29f0eeb50eacf3d2b] to archive 2024-12-05T00:44:06,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:44:06,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/5c4dea42ef7a4b16a172225fe0cca2cb to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/5c4dea42ef7a4b16a172225fe0cca2cb 2024-12-05T00:44:06,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/63303102c8f748c2b3c1e9ef62338454 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/63303102c8f748c2b3c1e9ef62338454 2024-12-05T00:44:06,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/019fb968ff394ed48ce87afcf0dc216b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/019fb968ff394ed48ce87afcf0dc216b 2024-12-05T00:44:06,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/41c76aad318a4209b023e7b620f42051 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/41c76aad318a4209b023e7b620f42051 2024-12-05T00:44:06,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/3295e927d746431f8130e5857ce02e97 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/3295e927d746431f8130e5857ce02e97 2024-12-05T00:44:06,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4b68218aa3674adf8b5850e8ea7a72e5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4b68218aa3674adf8b5850e8ea7a72e5 2024-12-05T00:44:06,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/54d2f9b4129c4139a8f3327c7da3d1e9 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/54d2f9b4129c4139a8f3327c7da3d1e9 2024-12-05T00:44:06,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/53115acdd7c047668573c886dca2499a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/53115acdd7c047668573c886dca2499a 2024-12-05T00:44:06,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/2664e9c2f09d4ab5ad7bf2aad7444e1a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/2664e9c2f09d4ab5ad7bf2aad7444e1a 2024-12-05T00:44:06,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/39636caefbaa44b987944f388baadaea to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/39636caefbaa44b987944f388baadaea 2024-12-05T00:44:06,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/e1a236d4099e45059b413bf09f31ab46 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/e1a236d4099e45059b413bf09f31ab46 2024-12-05T00:44:06,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/595fc3f66f7849c5b62a1f88ccac675f to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/595fc3f66f7849c5b62a1f88ccac675f 2024-12-05T00:44:06,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/20e415fbd78145c9a52b6149b74cfe2c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/20e415fbd78145c9a52b6149b74cfe2c 2024-12-05T00:44:06,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/7e868b655f7f4f1b8a2400bffd4c76ef to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/7e868b655f7f4f1b8a2400bffd4c76ef 2024-12-05T00:44:06,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4c42ce3a0d5942e986f3e5f6434b3c1d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/4c42ce3a0d5942e986f3e5f6434b3c1d 2024-12-05T00:44:06,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/91fdb434e9d348099f1fb97ffebbcb6b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/91fdb434e9d348099f1fb97ffebbcb6b 2024-12-05T00:44:06,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/6c0b1346ef1b42e29f0eeb50eacf3d2b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/6c0b1346ef1b42e29f0eeb50eacf3d2b 2024-12-05T00:44:06,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a5f52a842221473b9321ed80644b8dd0, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/128d0357f55e4f099fc8fb7a826c4215, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/0608aa2629f549eea7d17f0fe380415e, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2db5d1e6d5b5412b81ccf700e691f57c, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/f360daf4b1424b32bbe81459f38c6df6, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/53f28fe64a4b443ea91b3079462854c4, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c232737ee70f4d0b8a2364b197f7c762, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/edbcac339d6d482c964452ea581c6e34, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/71e5d8d152e643fa817a9cd9de165535, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/04b376e8e6954528b632095a4b9b3d3d, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/5c02a9265a0547949741631788848b74, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c6598abb5ffd4d97b1b049c28536f2c1, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/30b9402cf5da49cebbc32a94fbf7e76b, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/b9e9406017af43ec96297b839c8c5f54, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/6b266f6efba342d19505050aa6bca842, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/eed5b60a3fc6496c86bb63628b2a4e9e] to archive 2024-12-05T00:44:06,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T00:44:06,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a5f52a842221473b9321ed80644b8dd0 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a5f52a842221473b9321ed80644b8dd0 2024-12-05T00:44:06,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/128d0357f55e4f099fc8fb7a826c4215 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/128d0357f55e4f099fc8fb7a826c4215 2024-12-05T00:44:06,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/0608aa2629f549eea7d17f0fe380415e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/0608aa2629f549eea7d17f0fe380415e 2024-12-05T00:44:06,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2db5d1e6d5b5412b81ccf700e691f57c to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2db5d1e6d5b5412b81ccf700e691f57c 2024-12-05T00:44:06,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/f360daf4b1424b32bbe81459f38c6df6 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/f360daf4b1424b32bbe81459f38c6df6 2024-12-05T00:44:06,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/2e8bdaa16f6d4c7ba2b4ab3a6831d4b5 2024-12-05T00:44:06,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/53f28fe64a4b443ea91b3079462854c4 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/53f28fe64a4b443ea91b3079462854c4 2024-12-05T00:44:06,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c232737ee70f4d0b8a2364b197f7c762 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c232737ee70f4d0b8a2364b197f7c762 2024-12-05T00:44:06,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/edbcac339d6d482c964452ea581c6e34 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/edbcac339d6d482c964452ea581c6e34 2024-12-05T00:44:06,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/71e5d8d152e643fa817a9cd9de165535 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/71e5d8d152e643fa817a9cd9de165535 2024-12-05T00:44:06,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/04b376e8e6954528b632095a4b9b3d3d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/04b376e8e6954528b632095a4b9b3d3d 2024-12-05T00:44:06,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/5c02a9265a0547949741631788848b74 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/5c02a9265a0547949741631788848b74 2024-12-05T00:44:06,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c6598abb5ffd4d97b1b049c28536f2c1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/c6598abb5ffd4d97b1b049c28536f2c1 2024-12-05T00:44:06,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/30b9402cf5da49cebbc32a94fbf7e76b to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/30b9402cf5da49cebbc32a94fbf7e76b 2024-12-05T00:44:06,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/b9e9406017af43ec96297b839c8c5f54 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/b9e9406017af43ec96297b839c8c5f54 2024-12-05T00:44:06,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/6b266f6efba342d19505050aa6bca842 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/6b266f6efba342d19505050aa6bca842 2024-12-05T00:44:06,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/eed5b60a3fc6496c86bb63628b2a4e9e to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/eed5b60a3fc6496c86bb63628b2a4e9e 2024-12-05T00:44:06,862 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/recovered.edits/287.seqid, newMaxSeqId=287, maxSeqId=4 2024-12-05T00:44:06,862 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. 2024-12-05T00:44:06,863 DEBUG [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 6544d6803c2ff354880c0eae0e049fdc: Waiting for close lock at 1733359445113Running coprocessor pre-close hooks at 1733359445113Disabling compacts and flushes for region at 1733359445113Disabling writes for close at 1733359445113Obtaining lock to block concurrent updates at 1733359445114 (+1 ms)Preparing flush snapshotting stores in 6544d6803c2ff354880c0eae0e049fdc at 1733359445114Finished memstore snapshotting TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733359445115 (+1 ms)Flushing stores of TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc. at 1733359445116 (+1 ms)Flushing 6544d6803c2ff354880c0eae0e049fdc/A: creating writer at 1733359445116Flushing mob file 6544d6803c2ff354880c0eae0e049fdc/A: appending metadata at 1733359445125 (+9 ms)Flushing mob file 6544d6803c2ff354880c0eae0e049fdc/A: closing flushed file at 1733359445125Flushing 6544d6803c2ff354880c0eae0e049fdc/A: appending metadata at 1733359445539 (+414 ms)Flushing 6544d6803c2ff354880c0eae0e049fdc/A: closing flushed file at 1733359445539Flushing 6544d6803c2ff354880c0eae0e049fdc/B: creating writer at 1733359445956 (+417 ms)Flushing 6544d6803c2ff354880c0eae0e049fdc/B: appending metadata at 1733359445960 (+4 ms)Flushing 6544d6803c2ff354880c0eae0e049fdc/B: closing flushed file at 1733359445960Flushing 6544d6803c2ff354880c0eae0e049fdc/C: creating writer at 1733359446367 (+407 ms)Flushing 6544d6803c2ff354880c0eae0e049fdc/C: appending metadata at 1733359446370 (+3 ms)Flushing 6544d6803c2ff354880c0eae0e049fdc/C: closing flushed file at 1733359446370Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fadc35f: reopening flushed file at 1733359446783 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a16ecc0: reopening flushed file at 1733359446788 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36115ef0: reopening flushed file at 1733359446792 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6544d6803c2ff354880c0eae0e049fdc in 1684ms, sequenceid=284, compaction requested=true at 1733359446797 (+5 ms)Writing region close event to WAL at 1733359446860 (+63 ms)Running coprocessor post-close hooks at 1733359446862 (+2 ms)Closed at 1733359446862 2024-12-05T00:44:06,864 INFO [RS_CLOSE_REGION-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:06,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=6544d6803c2ff354880c0eae0e049fdc, regionState=CLOSED 2024-12-05T00:44:06,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 because future has completed 2024-12-05T00:44:06,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-05T00:44:06,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 6544d6803c2ff354880c0eae0e049fdc, server=fea72ea5c4b6,42683,1733359267474 in 1.9060 sec 2024-12-05T00:44:06,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-05T00:44:06,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6544d6803c2ff354880c0eae0e049fdc, UNASSIGN in 1.9100 sec 2024-12-05T00:44:06,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-05T00:44:06,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9130 sec 2024-12-05T00:44:06,871 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733359446871"}]},"ts":"1733359446871"} 2024-12-05T00:44:06,872 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T00:44:06,872 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T00:44:06,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9210 sec 2024-12-05T00:44:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-05T00:44:07,103 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:44:07,104 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T00:44:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:07,109 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T00:44:07,111 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:07,114 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,117 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C, FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/recovered.edits] 2024-12-05T00:44:07,121 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/16af0d3b7eff47b382865512c124b1e1 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/16af0d3b7eff47b382865512c124b1e1 2024-12-05T00:44:07,123 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/525caec657db4bd284368d38d52ab667 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/525caec657db4bd284368d38d52ab667 2024-12-05T00:44:07,124 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/aad283b4a4ec4577a999fc8f1a7d79e2 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/A/aad283b4a4ec4577a999fc8f1a7d79e2 2024-12-05T00:44:07,127 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/107773bebaac4bbc9b7678e5a93153da to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/107773bebaac4bbc9b7678e5a93153da 2024-12-05T00:44:07,129 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/b062c794b0bf4d5592f3d2972a3416b3 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/b062c794b0bf4d5592f3d2972a3416b3 2024-12-05T00:44:07,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/fe64f3cbdf614d03ac1543feb2ef908d to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/B/fe64f3cbdf614d03ac1543feb2ef908d 2024-12-05T00:44:07,133 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/1ed4d3f3f6ba4011b9482c8591bf133a to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/1ed4d3f3f6ba4011b9482c8591bf133a 2024-12-05T00:44:07,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a3872d52f5174a5aac30ffe575cd9a22 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/a3872d52f5174a5aac30ffe575cd9a22 2024-12-05T00:44:07,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/ab9cea76ee184af99bafa8ba13e054f7 to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/C/ab9cea76ee184af99bafa8ba13e054f7 2024-12-05T00:44:07,136 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/recovered.edits/287.seqid to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc/recovered.edits/287.seqid 2024-12-05T00:44:07,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/default/TestAcidGuarantees/6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,137 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T00:44:07,137 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T00:44:07,137 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T00:44:07,139 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051f0506db12f64048bb38ca64c2562728_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051f0506db12f64048bb38ca64c2562728_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,140 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052693a3dde2ed47ca86b7476827275455_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052693a3dde2ed47ca86b7476827275455_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,140 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120528b167b67bfc4a388d31567ce91a52cc_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120528b167b67bfc4a388d31567ce91a52cc_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,141 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c68087da67545309295dd72efcc5e18_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c68087da67545309295dd72efcc5e18_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,143 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120544a6724cacaa48bf843b7378d90161a6_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120544a6724cacaa48bf843b7378d90161a6_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,144 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205493d93bccbaf42c99f8706b0863c132c_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205493d93bccbaf42c99f8706b0863c132c_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,144 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054cc5a4d00bf64a78800d3cdffd40eb09_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054cc5a4d00bf64a78800d3cdffd40eb09_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,145 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120560d6e16655414cd89b41146e6eda0e76_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120560d6e16655414cd89b41146e6eda0e76_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,145 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056d8a4ac0da4642169c2fd9815143edd3_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056d8a4ac0da4642169c2fd9815143edd3_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,146 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205787e7f83329248f1aac995bc452aa418_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205787e7f83329248f1aac995bc452aa418_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,146 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a97595fe8f5b4510847ebec157c8f9ac_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a97595fe8f5b4510847ebec157c8f9ac_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,147 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1e47ef5d2584344a5ae3ea30b0d3fdc_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1e47ef5d2584344a5ae3ea30b0d3fdc_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,148 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c9a4c4bad3b84dd38e3b319c60aa182b_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c9a4c4bad3b84dd38e3b319c60aa182b_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,148 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d3dbe6255f5b41feb156a57458e133b9_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d3dbe6255f5b41feb156a57458e133b9_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,149 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f8e9ddfd2677459b980788b025ee063e_6544d6803c2ff354880c0eae0e049fdc to hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f8e9ddfd2677459b980788b025ee063e_6544d6803c2ff354880c0eae0e049fdc 2024-12-05T00:44:07,150 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T00:44:07,151 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:07,154 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T00:44:07,155 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T00:44:07,156 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:07,156 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T00:44:07,156 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733359447156"}]},"ts":"9223372036854775807"} 2024-12-05T00:44:07,158 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T00:44:07,158 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6544d6803c2ff354880c0eae0e049fdc, NAME => 'TestAcidGuarantees,,1733359414479.6544d6803c2ff354880c0eae0e049fdc.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T00:44:07,158 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T00:44:07,158 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733359447158"}]},"ts":"9223372036854775807"} 2024-12-05T00:44:07,159 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T00:44:07,160 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T00:44:07,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-05T00:44:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-05T00:44:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T00:44:07,223 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T00:44:07,238 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 451), ProcessCount=11 (was 11), AvailableMemoryMB=9310 (was 9325) 2024-12-05T00:44:07,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T00:44:07,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:07,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:07,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:07,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:07,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T00:44:07,239 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T00:44:07,239 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-12-05T00:44:07,239 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fea72ea5c4b6,41705,1733359266486 2024-12-05T00:44:07,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:07,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T00:44:07,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:07,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:07,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T00:44:07,324 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T00:44:07,324 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:07,324 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:07,324 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T00:44:07,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:07,325 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fea72ea5c4b6,42683,1733359267474' ***** 2024-12-05T00:44:07,325 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T00:44:07,326 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T00:44:07,326 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T00:44:07,326 INFO [RS:0;fea72ea5c4b6:42683 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(959): stopping server fea72ea5c4b6,42683,1733359267474 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fea72ea5c4b6:42683. 2024-12-05T00:44:07,327 DEBUG [RS:0;fea72ea5c4b6:42683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T00:44:07,327 DEBUG [RS:0;fea72ea5c4b6:42683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T00:44:07,327 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T00:44:07,328 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T00:44:07,328 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T00:44:07,329 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:07,329 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T00:44:07,329 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T00:44:07,329 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T00:44:07,329 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T00:44:07,329 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T00:44:07,329 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-05T00:44:07,350 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/info/4691760f01924844895503ecd62ee793 is 95, key is TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0./info:/1733359300037/DeleteFamily/seqid=0 2024-12-05T00:44:07,353 INFO [regionserver/fea72ea5c4b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:07,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742397_1573 (size=6514) 2024-12-05T00:44:07,529 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:07,729 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:07,755 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/info/4691760f01924844895503ecd62ee793 2024-12-05T00:44:07,762 INFO [regionserver/fea72ea5c4b6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T00:44:07,762 INFO [regionserver/fea72ea5c4b6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T00:44:07,779 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/ns/7e93e492d9064509b4c3fdbe2a7cbefc is 93, key is TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0./ns:/1733359299997/DeleteFamily/seqid=0 2024-12-05T00:44:07,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742398_1574 (size=5906) 2024-12-05T00:44:07,930 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:08,130 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:08,183 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/ns/7e93e492d9064509b4c3fdbe2a7cbefc 2024-12-05T00:44:08,213 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/rep_barrier/ae19a614bb1d4671b86d1baedd8a2949 is 102, key is TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0./rep_barrier:/1733359299997/DeleteFamily/seqid=0 2024-12-05T00:44:08,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742399_1575 (size=6025) 2024-12-05T00:44:08,330 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:08,531 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:08,618 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/rep_barrier/ae19a614bb1d4671b86d1baedd8a2949 2024-12-05T00:44:08,644 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/table/7bb77ba3b5cf4edcb76d7c55484abdb9 is 96, key is TestAcidGuarantees,,1733359271834.d5061eee39269f2ef93c654f6e1034a0./table:/1733359299997/DeleteFamily/seqid=0 2024-12-05T00:44:08,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742400_1576 (size=5936) 2024-12-05T00:44:08,731 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:08,931 DEBUG [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T00:44:09,049 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/table/7bb77ba3b5cf4edcb76d7c55484abdb9 2024-12-05T00:44:09,057 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/info/4691760f01924844895503ecd62ee793 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/info/4691760f01924844895503ecd62ee793 2024-12-05T00:44:09,061 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/info/4691760f01924844895503ecd62ee793, entries=12, sequenceid=90, filesize=6.4 K 2024-12-05T00:44:09,062 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/ns/7e93e492d9064509b4c3fdbe2a7cbefc as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/ns/7e93e492d9064509b4c3fdbe2a7cbefc 2024-12-05T00:44:09,066 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/ns/7e93e492d9064509b4c3fdbe2a7cbefc, entries=8, sequenceid=90, filesize=5.8 K 2024-12-05T00:44:09,067 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/rep_barrier/ae19a614bb1d4671b86d1baedd8a2949 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/rep_barrier/ae19a614bb1d4671b86d1baedd8a2949 2024-12-05T00:44:09,071 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/rep_barrier/ae19a614bb1d4671b86d1baedd8a2949, entries=6, sequenceid=90, filesize=5.9 K 2024-12-05T00:44:09,072 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/.tmp/table/7bb77ba3b5cf4edcb76d7c55484abdb9 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/table/7bb77ba3b5cf4edcb76d7c55484abdb9 2024-12-05T00:44:09,075 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/table/7bb77ba3b5cf4edcb76d7c55484abdb9, entries=7, sequenceid=90, filesize=5.8 K 2024-12-05T00:44:09,076 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1747ms, sequenceid=90, compaction requested=false 2024-12-05T00:44:09,080 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-05T00:44:09,081 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T00:44:09,081 INFO [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:09,081 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733359447328Running coprocessor pre-close hooks at 1733359447329 (+1 ms)Disabling compacts and flushes for region at 1733359447329Disabling writes for close at 1733359447329Obtaining lock to block concurrent updates at 1733359447329Preparing flush snapshotting stores in 1588230740 at 1733359447329Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733359447330 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733359447330Flushing 1588230740/info: creating writer at 1733359447330Flushing 1588230740/info: appending metadata at 1733359447350 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733359447350Flushing 1588230740/ns: creating writer at 1733359447763 (+413 ms)Flushing 1588230740/ns: appending metadata at 1733359447779 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733359447779Flushing 1588230740/rep_barrier: creating writer at 1733359448188 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733359448213 (+25 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733359448213Flushing 1588230740/table: creating writer at 1733359448626 (+413 ms)Flushing 1588230740/table: appending metadata at 1733359448643 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733359448643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bdbb4af: reopening flushed file at 1733359449056 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51853e5a: reopening flushed file at 1733359449062 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dd64f1: reopening flushed file at 1733359449067 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c7351db: reopening flushed file at 1733359449071 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1747ms, sequenceid=90, compaction requested=false at 1733359449076 (+5 ms)Writing region close event to WAL at 1733359449077 (+1 ms)Running coprocessor post-close hooks at 1733359449081 (+4 ms)Closed at 1733359449081 2024-12-05T00:44:09,081 DEBUG [RS_CLOSE_META-regionserver/fea72ea5c4b6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T00:44:09,132 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(976): stopping server fea72ea5c4b6,42683,1733359267474; all regions closed. 2024-12-05T00:44:09,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741834_1010 (size=25344) 2024-12-05T00:44:09,144 DEBUG [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/oldWALs 2024-12-05T00:44:09,144 INFO [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C42683%2C1733359267474.meta:.meta(num 1733359271328) 2024-12-05T00:44:09,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741833_1009 (size=14150879) 2024-12-05T00:44:09,148 DEBUG [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/oldWALs 2024-12-05T00:44:09,148 INFO [RS:0;fea72ea5c4b6:42683 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fea72ea5c4b6%2C42683%2C1733359267474:(num 1733359270842) 2024-12-05T00:44:09,148 DEBUG [RS:0;fea72ea5c4b6:42683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T00:44:09,148 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T00:44:09,148 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:09,148 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.ChoreService(370): Chore service for: regionserver/fea72ea5c4b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:09,149 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:09,149 INFO [regionserver/fea72ea5c4b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:09,149 INFO [RS:0;fea72ea5c4b6:42683 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42683 2024-12-05T00:44:09,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T00:44:09,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fea72ea5c4b6,42683,1733359267474 2024-12-05T00:44:09,217 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:09,218 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fea72ea5c4b6,42683,1733359267474] 2024-12-05T00:44:09,317 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fea72ea5c4b6,42683,1733359267474 already deleted, retry=false 2024-12-05T00:44:09,318 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fea72ea5c4b6,42683,1733359267474 expired; onlineServers=0 2024-12-05T00:44:09,318 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fea72ea5c4b6,41705,1733359266486' ***** 2024-12-05T00:44:09,318 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T00:44:09,318 INFO [M:0;fea72ea5c4b6:41705 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T00:44:09,318 INFO [M:0;fea72ea5c4b6:41705 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T00:44:09,318 DEBUG [M:0;fea72ea5c4b6:41705 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T00:44:09,319 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T00:44:09,319 DEBUG [M:0;fea72ea5c4b6:41705 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T00:44:09,319 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359270433 {}] cleaner.HFileCleaner(306): Exit Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.large.0-1733359270433,5,FailOnTimeoutGroup] 2024-12-05T00:44:09,319 DEBUG [master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359270436 {}] cleaner.HFileCleaner(306): Exit Thread[master/fea72ea5c4b6:0:becomeActiveMaster-HFileCleaner.small.0-1733359270436,5,FailOnTimeoutGroup] 2024-12-05T00:44:09,319 INFO [M:0;fea72ea5c4b6:41705 {}] hbase.ChoreService(370): Chore service for: master/fea72ea5c4b6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T00:44:09,319 INFO [M:0;fea72ea5c4b6:41705 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T00:44:09,319 DEBUG [M:0;fea72ea5c4b6:41705 {}] master.HMaster(1795): Stopping service threads 2024-12-05T00:44:09,319 INFO [M:0;fea72ea5c4b6:41705 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T00:44:09,320 INFO [M:0;fea72ea5c4b6:41705 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T00:44:09,320 ERROR [M:0;fea72ea5c4b6:41705 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-05T00:44:09,321 INFO [M:0;fea72ea5c4b6:41705 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T00:44:09,321 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T00:44:09,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T00:44:09,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T00:44:09,332 DEBUG [M:0;fea72ea5c4b6:41705 {}] zookeeper.ZKUtil(347): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T00:44:09,332 WARN [M:0;fea72ea5c4b6:41705 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T00:44:09,333 INFO [M:0;fea72ea5c4b6:41705 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/.lastflushedseqids 2024-12-05T00:44:09,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742401_1577 (size=72) 2024-12-05T00:44:09,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:09,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42683-0x101a2ef632c0001, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:09,392 INFO [RS:0;fea72ea5c4b6:42683 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:09,393 INFO [RS:0;fea72ea5c4b6:42683 {}] regionserver.HRegionServer(1031): Exiting; stopping=fea72ea5c4b6,42683,1733359267474; zookeeper connection closed. 2024-12-05T00:44:09,393 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3524a029 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3524a029 2024-12-05T00:44:09,394 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T00:44:09,749 INFO [M:0;fea72ea5c4b6:41705 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T00:44:09,750 INFO [M:0;fea72ea5c4b6:41705 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T00:44:09,750 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T00:44:09,751 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:09,751 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:09,751 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T00:44:09,751 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:09,751 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=764.54 KB heapSize=935.52 KB 2024-12-05T00:44:09,794 DEBUG [M:0;fea72ea5c4b6:41705 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddc46dc84f3e4b29a8ed3de7961aef78 is 82, key is hbase:meta,,1/info:regioninfo/1733359271415/Put/seqid=0 2024-12-05T00:44:09,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742402_1578 (size=5672) 2024-12-05T00:44:10,197 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2110 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddc46dc84f3e4b29a8ed3de7961aef78 2024-12-05T00:44:10,219 DEBUG [M:0;fea72ea5c4b6:41705 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7af299892c084d06bd9df729b6931d81 is 2480, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1733359302224/Put/seqid=0 2024-12-05T00:44:10,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742403_1579 (size=44727) 2024-12-05T00:44:10,625 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=763.99 KB at sequenceid=2110 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7af299892c084d06bd9df729b6931d81 2024-12-05T00:44:10,629 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7af299892c084d06bd9df729b6931d81 2024-12-05T00:44:10,644 DEBUG [M:0;fea72ea5c4b6:41705 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74eff49a629941f8b7d8998df5d847f8 is 69, key is fea72ea5c4b6,42683,1733359267474/rs:state/1733359270552/Put/seqid=0 2024-12-05T00:44:10,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073742404_1580 (size=5156) 2024-12-05T00:44:11,048 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2110 (bloomFilter=true), to=hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74eff49a629941f8b7d8998df5d847f8 2024-12-05T00:44:11,058 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddc46dc84f3e4b29a8ed3de7961aef78 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddc46dc84f3e4b29a8ed3de7961aef78 2024-12-05T00:44:11,061 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddc46dc84f3e4b29a8ed3de7961aef78, entries=8, sequenceid=2110, filesize=5.5 K 2024-12-05T00:44:11,062 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7af299892c084d06bd9df729b6931d81 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7af299892c084d06bd9df729b6931d81 2024-12-05T00:44:11,065 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7af299892c084d06bd9df729b6931d81 2024-12-05T00:44:11,065 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7af299892c084d06bd9df729b6931d81, entries=166, sequenceid=2110, filesize=43.7 K 2024-12-05T00:44:11,066 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74eff49a629941f8b7d8998df5d847f8 as hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74eff49a629941f8b7d8998df5d847f8 2024-12-05T00:44:11,069 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42819/user/jenkins/test-data/addb28e5-aced-933c-5ea5-d882489dea18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74eff49a629941f8b7d8998df5d847f8, entries=1, sequenceid=2110, filesize=5.0 K 2024-12-05T00:44:11,070 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(3140): Finished flush of dataSize ~764.54 KB/782894, heapSize ~935.23 KB/957672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1319ms, sequenceid=2110, compaction requested=false 2024-12-05T00:44:11,071 INFO [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T00:44:11,071 DEBUG [M:0;fea72ea5c4b6:41705 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733359449750Disabling compacts and flushes for region at 1733359449750Disabling writes for close at 1733359449751 (+1 ms)Obtaining lock to block concurrent updates at 1733359449751Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733359449751Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=782894, getHeapSize=957912, getOffHeapSize=0, getCellsCount=2301 at 1733359449752 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733359449775 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733359449775Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733359449794 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733359449794Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733359450201 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733359450218 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733359450219 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733359450629 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733359450644 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733359450644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b7a8e7: reopening flushed file at 1733359451057 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7416fcd5: reopening flushed file at 1733359451061 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c7b68f: reopening flushed file at 1733359451065 (+4 ms)Finished flush of dataSize ~764.54 KB/782894, heapSize ~935.23 KB/957672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1319ms, sequenceid=2110, compaction requested=false at 1733359451070 (+5 ms)Writing region close event to WAL at 1733359451071 (+1 ms)Closed at 1733359451071 2024-12-05T00:44:11,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41735 is added to blk_1073741830_1006 (size=921227) 2024-12-05T00:44:11,075 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T00:44:11,075 INFO [M:0;fea72ea5c4b6:41705 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T00:44:11,075 INFO [M:0;fea72ea5c4b6:41705 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41705 2024-12-05T00:44:11,075 INFO [M:0;fea72ea5c4b6:41705 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T00:44:11,201 INFO [M:0;fea72ea5c4b6:41705 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T00:44:11,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:11,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41705-0x101a2ef632c0000, quorum=127.0.0.1:50354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T00:44:11,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T00:44:11,214 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T00:44:11,214 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T00:44:11,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T00:44:11,215 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/hadoop.log.dir/,STOPPED} 2024-12-05T00:44:11,218 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T00:44:11,218 WARN [BP-1145415758-172.17.0.2-1733359262674 heartbeating to localhost/127.0.0.1:42819 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T00:44:11,218 WARN [BP-1145415758-172.17.0.2-1733359262674 heartbeating to localhost/127.0.0.1:42819 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1145415758-172.17.0.2-1733359262674 (Datanode Uuid e92b3860-bd6b-47f0-9a8c-8a2b489e331f) service to localhost/127.0.0.1:42819 2024-12-05T00:44:11,218 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T00:44:11,219 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/data/data1/current/BP-1145415758-172.17.0.2-1733359262674 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T00:44:11,219 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/cluster_5a8d5751-cfd9-8f41-eb6e-72a31b25fc6c/data/data2/current/BP-1145415758-172.17.0.2-1733359262674 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T00:44:11,220 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T00:44:11,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T00:44:11,226 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T00:44:11,226 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T00:44:11,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T00:44:11,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a2e5686-146f-8b72-9659-3f4913ea3c7f/hadoop.log.dir/,STOPPED} 2024-12-05T00:44:11,240 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T00:44:11,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down